From 716f36d14deb1540b8f6f8b05c9a6439239c0474 Mon Sep 17 00:00:00 2001 From: Christian Schmied Date: Thu, 26 Sep 2024 11:33:03 +0200 Subject: [PATCH] init --- .gitignore | 145 +++++++++++++++++++++++++++++++++++++++ ArchiverConfig.go | 6 ++ ErrorOnlyArchiver.go | 43 ++++++++++++ FileArchiver.go | 103 +++++++++++++++++++++++++++ FileArchiver_test.go | 111 ++++++++++++++++++++++++++++++ Globbing_linux_test.go | 26 +++++++ Globbing_windows_test.go | 20 ++++++ NoOpArchiver.go | 7 ++ getGlob_linux.go | 35 ++++++++++ getGlob_windows.go | 30 ++++++++ go.mod | 7 ++ go.sum | 15 ++++ 12 files changed, 548 insertions(+) create mode 100644 .gitignore create mode 100644 ArchiverConfig.go create mode 100644 ErrorOnlyArchiver.go create mode 100644 FileArchiver.go create mode 100644 FileArchiver_test.go create mode 100644 Globbing_linux_test.go create mode 100644 Globbing_windows_test.go create mode 100644 NoOpArchiver.go create mode 100644 getGlob_linux.go create mode 100644 getGlob_windows.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..067758e --- /dev/null +++ b/.gitignore @@ -0,0 +1,145 @@ +# Created by https://www.toptal.com/developers/gitignore/api/go,goland+all,linux,git +# Edit at https://www.toptal.com/developers/gitignore?templates=go,goland+all,linux,git + +### Git ### +# Created by git for backups. To disable backups in Git: +# $ git config --global mergetool.keepBackup false +*.orig + +# Created by git when using merge tools for conflicts +*.BACKUP.* +*.BASE.* +*.LOCAL.* +*.REMOTE.* +*_BACKUP_*.txt +*_BASE_*.txt +*_LOCAL_*.txt +*_REMOTE_*.txt + +### Go ### +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +### GoLand+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### GoLand+all Patch ### +# Ignore everything but code style settings and run configurations +# that are supposed to be shared within teams. + +.idea/* + +!.idea/codeStyles +!.idea/runConfigurations + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +# End of https://www.toptal.com/developers/gitignore/api/go,goland+all,linux,git diff --git a/ArchiverConfig.go b/ArchiverConfig.go new file mode 100644 index 0000000..3ab1974 --- /dev/null +++ b/ArchiverConfig.go @@ -0,0 +1,6 @@ +package archiver + +type Config struct { + Folder string + ArchiveFolder string +} diff --git a/ErrorOnlyArchiver.go b/ErrorOnlyArchiver.go new file mode 100644 index 0000000..20df8b1 --- /dev/null +++ b/ErrorOnlyArchiver.go @@ -0,0 +1,43 @@ +package archiver + +import ( + "os" + "path/filepath" +) + +func NewErrorOnlyArchiver(conf *Config) func(string, error) { + return func(filePath string, archiveErr error) { + if archiveErr != nil { + err := writeErrorFile(conf, filePath, archiveErr) + logger.Error(err) + } + } +} + +func writeErrorFile(conf *Config, filePath string, archiveErr error) error { + if archiveErr != nil { + baseName := filepath.Base(filePath) + + destFolder := getDestFolder(conf, ErrorFolder) + globDir := extractGlobbedDir(conf.Folder, filePath) + if globDir != "" { + destFolder = filepath.Join(destFolder, globDir) + } + + err := os.MkdirAll(destFolder, os.ModePerm) + if err != nil { + return err + } + + f, err := os.OpenFile(filepath.Join(destFolder, baseName+".err"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.ModePerm) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write([]byte(archiveErr.Error() + "\n")) + if err != nil { + return err + } + } + return nil +} diff --git a/FileArchiver.go b/FileArchiver.go new file mode 100644 index 0000000..92da89d --- /dev/null +++ b/FileArchiver.go @@ -0,0 +1,103 @@ +package archiver + +import ( + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "strings" + "time" +) + +const _YEARMONTHONLY = "200601" + +const ArchiveFolder = "archive" +const ErrorFolder = "error" + +var logger = logrus.WithField("package", "archiver") + +func NewFileArchiver(conf *Config) func(string, error) { + return func(f string, archiveErr error) { + var err error = nil + if archiveErr != nil { + err = moveToError(conf, f, archiveErr) + } else { + err = moveToArchive(conf, f) + } + + if err != nil { + logger.Error(err) + } + } +} + +func extractGlobbedDir(rootPath string, filePath string) string { + if !strings.Contains(rootPath, "*") { + return "" + } + fileDir := filepath.Dir(filePath) + globRoot := getGlobRoot(rootPath) + globbedDir, err := filepath.Rel(globRoot, fileDir) + if err != nil { + logger. + WithField("rootPath", rootPath). + WithField("globRoot", globRoot). + WithField("fileDir", fileDir). + Error(err) + return "" + } + return globbedDir +} +func getDestFolder(conf *Config, archiveFolder string) string { + var destFolder string + if len(conf.ArchiveFolder) > 0 { + destFolder = filepath.Join(conf.ArchiveFolder) + } else { + destFolder = filepath.Join(conf.Folder) + destFolder = getGlobRoot(destFolder) + } + return filepath.Join(destFolder, archiveFolder) +} + +func moveToArchive(conf *Config, filePath string) error { + baseName := filepath.Base(filePath) + dateString := time.Now().Format(_YEARMONTHONLY) + + destFolder := getDestFolder(conf, ArchiveFolder) + globDir := extractGlobbedDir(conf.Folder, filePath) + if globDir != "" { + destFolder = filepath.Join(destFolder, globDir) + } + + destFolder = filepath.Join(destFolder, dateString) + + err := os.MkdirAll(destFolder, os.ModePerm) + if err != nil { + return err + } + err = os.Rename(filePath, filepath.Join(destFolder, baseName)) + if err != nil { + return err + } + return nil +} + +func moveToError(conf *Config, filePath string, archiveErr error) error { + baseName := filepath.Base(filePath) + + destFolder := getDestFolder(conf, ErrorFolder) + globDir := extractGlobbedDir(conf.Folder, filePath) + if globDir != "" { + destFolder = filepath.Join(destFolder, globDir) + } + + err := os.MkdirAll(destFolder, os.ModePerm) + if err != nil { + return err + } + err = os.Rename(filePath, filepath.Join(destFolder, baseName)) + err = writeErrorFile(conf, filePath, archiveErr) + if err != nil { + return err + } + return nil +} diff --git a/FileArchiver_test.go b/FileArchiver_test.go new file mode 100644 index 0000000..5ce828f --- /dev/null +++ b/FileArchiver_test.go @@ -0,0 +1,111 @@ +package archiver + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" +) + +func TestFileArchiver(t *testing.T) { + tmpPath, err := os.MkdirTemp("", ".filearchiver_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpPath) + + conf := Config{ + Folder: tmpPath, + ArchiveFolder: "", + } + f := NewFileArchiver(&conf) + + okFile := filepath.Join(tmpPath, "ok.txt") + _ = os.WriteFile(okFile, []byte("test"), 0644) + f(okFile, nil) + + _, err = os.Stat(okFile) + if !os.IsNotExist(err) { + t.Errorf("file %s still exists", okFile) + } + archivedPath := filepath.Join(tmpPath, ArchiveFolder, time.Now().Format(_YEARMONTHONLY), "ok.txt") + _, err = os.Stat(archivedPath) + if os.IsNotExist(err) { + t.Errorf("file %s does not Exist", archivedPath) + } + + failFile := filepath.Join(tmpPath, "fail.txt") + _ = os.WriteFile(failFile, []byte("test"), 0644) + f(failFile, fmt.Errorf("some failure")) + + _, err = os.Stat(failFile) + if !os.IsNotExist(err) { + t.Errorf("file %s still exists", failFile) + } + _, err = os.Stat(filepath.Join(tmpPath, ArchiveFolder, time.Now().Format(_YEARMONTHONLY), "fail.txt")) + if !os.IsNotExist(err) { + t.Errorf("file %s is in Archive", failFile) + } + erroredPath := filepath.Join(tmpPath, ErrorFolder, "fail.txt") + _, err = os.Stat(erroredPath) + if os.IsNotExist(err) { + t.Errorf("file %s does not Exist", erroredPath) + } + _, err = os.Stat(filepath.Join(tmpPath, ErrorFolder, "fail.txt.err")) + if os.IsNotExist(err) { + t.Errorf("file fail.txt.err is not in Error") + } +} + +func TestGlobbedArchiver(t *testing.T) { + tmpPath, err := os.MkdirTemp("", ".filearchiver_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpPath) + + conf := Config{ + Folder: filepath.Join(tmpPath, "*"), + ArchiveFolder: "", + } + f := NewFileArchiver(&conf) + + _ = os.MkdirAll(filepath.Join(tmpPath, "foo"), os.ModePerm) + okFile := filepath.Join(tmpPath, "foo", "ok.txt") + _ = os.WriteFile(okFile, []byte("test"), 0644) + f(okFile, nil) + + _, err = os.Stat(okFile) + if !os.IsNotExist(err) { + t.Errorf("file %s still exists", okFile) + } + archivedPath := filepath.Join(tmpPath, ArchiveFolder, "foo", time.Now().Format(_YEARMONTHONLY), "ok.txt") + _, err = os.Stat(archivedPath) + if os.IsNotExist(err) { + t.Errorf("file %s does not Exist", archivedPath) + } + + _ = os.MkdirAll(filepath.Join(tmpPath, "bar"), os.ModePerm) + failFile := filepath.Join(tmpPath, "bar", "fail.txt") + _ = os.WriteFile(failFile, []byte("test"), 0644) + f(failFile, fmt.Errorf("some failure")) + + _, err = os.Stat(failFile) + if !os.IsNotExist(err) { + t.Errorf("file %s still exists", failFile) + } + _, err = os.Stat(filepath.Join(tmpPath, ArchiveFolder, "bar", time.Now().Format(_YEARMONTHONLY), "fail.txt")) + if !os.IsNotExist(err) { + t.Errorf("file %s is in Archive", failFile) + } + erroredPath := filepath.Join(tmpPath, ErrorFolder, "bar", "fail.txt") + _, err = os.Stat(erroredPath) + if os.IsNotExist(err) { + t.Errorf("file %s does not Exist", erroredPath) + } + _, err = os.Stat(filepath.Join(tmpPath, ErrorFolder, "bar", "fail.txt.err")) + if os.IsNotExist(err) { + t.Errorf("file fail.txt.err is not in Error") + } +} diff --git a/Globbing_linux_test.go b/Globbing_linux_test.go new file mode 100644 index 0000000..26095e4 --- /dev/null +++ b/Globbing_linux_test.go @@ -0,0 +1,26 @@ +package archiver + +import "testing" + +func TestGetGlobRoot(t *testing.T) { + assertThat(t, getGlobRoot("/foo/bar/baz"), "/foo/bar/baz") + + assertThat(t, getGlobRoot("/foo/bar/*/baz"), "/foo/bar") + assertThat(t, getGlobRoot("/foo/bar/*"), "/foo/bar") + + assertThat(t, getGlobRoot("foo/bar/*"), "foo/bar") + assertThat(t, getGlobRoot("./foo/bar/*"), "./foo/bar") +} + +func TestGetGlobbedDir(t *testing.T) { + assertThat(t, extractGlobbedDir("/foo/bar/baz", "/foo/bar/baz/myFile.txt"), "") + + assertThat(t, extractGlobbedDir("/foo/bar/*", "/foo/bar/myDir/myFile.txt"), "myDir") + assertThat(t, extractGlobbedDir("/foo/bar/*/baz", "/foo/bar/myDir/baz/myFile.txt"), "myDir/baz") +} +func assertThat[E comparable](t *testing.T, actual E, expected E) { + t.Helper() + if actual != expected { + t.Errorf("actual %v, expected %v", actual, expected) + } +} diff --git a/Globbing_windows_test.go b/Globbing_windows_test.go new file mode 100644 index 0000000..82da3f1 --- /dev/null +++ b/Globbing_windows_test.go @@ -0,0 +1,20 @@ +package archiver + +import ( + "testing" +) + +func TestGetGlobRoot(t *testing.T) { + assertThat(t, getGlobRoot("C:\\foo\\bar\\baz"), "C:\\foo\\bar\\baz") + + assertThat(t, getGlobRoot("C:\\foo\\bar\\*\\baz"), "C:\\foo\\bar") + assertThat(t, getGlobRoot("C:\\foo\\bar\\*"), "C:\\foo\\bar") + + assertThat(t, getGlobRoot("foo\\bar\\*"), "foo\\bar") +} +func assertThat[E comparable](t *testing.T, actual E, expected E) { + t.Helper() + if actual != expected { + t.Errorf("actual %v, expected %v", actual, expected) + } +} diff --git a/NoOpArchiver.go b/NoOpArchiver.go new file mode 100644 index 0000000..9ffdb68 --- /dev/null +++ b/NoOpArchiver.go @@ -0,0 +1,7 @@ +package archiver + +func NewNoOpArchiver() func(string, error) { + return func(string, error) { + + } +} diff --git a/getGlob_linux.go b/getGlob_linux.go new file mode 100644 index 0000000..7a6371e --- /dev/null +++ b/getGlob_linux.go @@ -0,0 +1,35 @@ +package archiver + +import ( + "path/filepath" + "strings" +) + +func getGlobRoot(path string) string { + // Splitte den Pfad in seine Komponenten + pathSplit := strings.Split(path, string(filepath.Separator)) + + // Durchlaufe die Pfadsegmente, um nach dem ersten '*' zu suchen + for i, split := range pathSplit { + if strings.Contains(split, "*") { + // Kombiniere den Pfad bis zum ersten '*' Segment + combined := filepath.Join(pathSplit[0:i]...) + + // Falls der Pfad absolut ist, füge den führenden Separator hinzu + if filepath.IsAbs(path) { + return string(filepath.Separator) + combined + } + + // Falls das erste Segment ein "." ist, gib den relativen Pfad zurück + if pathSplit[0] == "." { + return "." + string(filepath.Separator) + combined + } + + // Andernfalls gib den kombinierten Pfad ohne Anpassung zurück + return combined + } + } + + // Falls kein '*' gefunden wurde, gib den originalen Pfad zurück + return path +} diff --git a/getGlob_windows.go b/getGlob_windows.go new file mode 100644 index 0000000..88f9e4c --- /dev/null +++ b/getGlob_windows.go @@ -0,0 +1,30 @@ +package archiver + +import ( + "path/filepath" + "strings" +) + +func getGlobRoot(path string) string { + // Pfad säubern und Laufwerksbuchstaben trennen + cleanedPath := filepath.Clean(path) + volume := filepath.VolumeName(cleanedPath) // Erfasst Laufwerksbuchstaben wie "C:" + + // Pfad ohne Volume-Name splitten + remainingPath := strings.TrimPrefix(cleanedPath, volume) + pathSplit := strings.Split(remainingPath, string(filepath.Separator)) + + // Suche nach erstem "*" in den Pfadsegmenten + for i, split := range pathSplit { + if strings.Contains(split, "*") { + // Falls ein Volume-Name vorhanden ist, füge es korrekt hinzu + if len(volume) > 0 { + volumeRoot := volume + string(filepath.Separator) + return filepath.Join(append([]string{volumeRoot}, pathSplit[:i]...)...) + } + return filepath.Join(pathSplit[:i]...) + } + } + // Wenn kein "*" gefunden wurde, gib den gesamten Pfad zurück + return cleanedPath +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e1d5ee5 --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module git.sportwanninger.de/go/archiver + +go 1.23.1 + +require github.com/sirupsen/logrus v1.9.3 + +require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..21f9bfb --- /dev/null +++ b/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=