Skip to content

Commit

Permalink
fix issue #500 (#502)
Browse files Browse the repository at this point in the history 
* Refactor LocalAssetFile to add LogValue method

* Refactor readFolder.go to not group photo and move with a different capture date

* Refactor readFolder.go to improve logging of associated metadata

* Refactor log.go to separate log file opening logic

* readFolder: adapt tests to write a log

* Refactor metadata.go to add LogValue method

* Refactor log.go to use assetFile struct for logging in logMessage function

* Refactor json_test.go to include dateTaken and title fields in the test cases

* Refactor json.go to simplify the  GoogleMetaData struct, and fix the logValue()

* Refactor log.go to fanout log in file and console with different handler

* Refactor fileevent package to separate FileAndName struct and add LogValue method

* Refactor googlephotos.go to improve logging and metadata handling

* Fix json.go to handle photo taken time in GoogleMetaData struct

* Refactor googlephotos.go to handle photo taken time in GoogleMetaData struct

* Refactor uploadAsset method to improve error handling and logging

* Refactor upload/ui.go to use colored logs

* Refactor log.go to use console-slog for colored logs

* Refactor ui  to display google takeout progression

* Refactor error handling in main.go

* Update dependencies in go.mod
  • Loading branch information
@simulot
simulot authored Oct 13, 2024
1 parent b035542 commit 0bf90c2
Show file tree
Hide file tree
Showing 26 changed files with 636 additions and 276 deletions.
28 changes: 6 additions & 22 deletions adapters/folder/readFolder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,24 +197,7 @@ func (la *LocalAssetBrowser) passTwo(ctx context.Context) chan *adapters.AssetGr
links[base] = image
continue
}
/*
if image, ok := links[base]; ok {
// file.XMP -> file.ext
image.sidecar = file
links[base] = image
continue
}
for lk, l := range links {
f := l.image
if strings.TrimSuffix(f, path.Ext(f)) == file {
// file.ext.XMP -> file.ext
l.sidecar = file
links[lk] = l
continue
}

}
*/
case metadata.TypeVideo:
if image, ok := links[file]; ok {
// file.MP.jpg -> file.MP
Expand Down Expand Up @@ -272,7 +255,7 @@ func (la *LocalAssetBrowser) passTwo(ctx context.Context) chan *adapters.AssetGr
continue
}
if a != nil && linked.sidecar != "" {
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, fileevent.AsFileAndName(fsys, a.FileName), "sidecar", linked.sidecar)
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, a, "sidecar", linked.sidecar)
a.SideCar = metadata.SideCarFile{
FSys: fsys,
FileName: linked.sidecar,
Expand All @@ -290,7 +273,7 @@ func (la *LocalAssetBrowser) passTwo(ctx context.Context) chan *adapters.AssetGr
continue
}
if linked.sidecar != "" {
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, fileevent.AsFileAndName(fsys, a.FileName), "sidecar", linked.sidecar)
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, a, "sidecar", linked.sidecar)
a.SideCar = metadata.SideCarFile{
FSys: fsys,
FileName: linked.sidecar,
Expand All @@ -306,7 +289,7 @@ func (la *LocalAssetBrowser) passTwo(ctx context.Context) chan *adapters.AssetGr
continue
}
if linked.sidecar != "" {
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, fileevent.AsFileAndName(fsys, a.FileName), "sidecar", linked.sidecar)
la.log.Record(ctx, fileevent.AnalysisAssociatedMetadata, a, "sidecar", linked.sidecar)
a.SideCar = metadata.SideCarFile{
FSys: fsys,
FileName: linked.sidecar,
Expand Down Expand Up @@ -373,9 +356,10 @@ func (la *LocalAssetBrowser) passTwo(ctx context.Context) chan *adapters.AssetGr
}
if abs(baseDate.Sub(aDate)) > 1*time.Minute {
// take this asset out of the group
g.Assets = append(g.Assets[:i], g.Assets[i+1:]...)
g.Assets = append(g.Assets[:i+1], g.Assets[i+2:]...)
// create a group for this assed
g2 := adapters.NewAssetGroup(adapters.GroupKindNone, a)
g2.Albums = g.Albums
gs = append(gs, g2)
}
}
Expand Down Expand Up @@ -432,7 +416,7 @@ func (la *LocalAssetBrowser) assetFromFile(ctx context.Context, fsys fs.FS, name

if la.flags.InclusionFlags.DateRange.IsSet() && !la.flags.InclusionFlags.DateRange.InRange(a.CaptureDate) {
a.Close()
la.log.Record(ctx, fileevent.DiscoveredDiscarded, fileevent.AsFileAndName(fsys, name), "reason", "asset outside date range")
la.log.Record(ctx, fileevent.DiscoveredDiscarded, a, "reason", "asset outside date range")
return nil, nil
}
return a, nil
Expand Down
33 changes: 24 additions & 9 deletions adapters/folder/e2e_readFolder_test.go → adapters/folder/readFolderWithFIles_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@ import (
"time"

"github.com/kr/pretty"
"github.com/simulot/immich-go/commands/application"
"github.com/simulot/immich-go/helpers/configuration"
cliflags "github.com/simulot/immich-go/internal/cliFlags"
"github.com/simulot/immich-go/internal/fileevent"
"github.com/simulot/immich-go/internal/metadata"
"github.com/simulot/immich-go/internal/tzone"
)

func TestE2ELocalAssets(t *testing.T) {
func TestLocalAssets(t *testing.T) {
tc := []struct {
name string
fsys []fs.FS
Expand Down Expand Up @@ -45,7 +47,7 @@ func TestE2ELocalAssets(t *testing.T) {
"photo1_2024-10-06_w_exif.jpg": {image: "photo1_2024-10-06_w_exif.jpg"},
"photo1_2023-10-06_wo_exif.jpg": {image: "photo1_2023-10-06_wo_exif.jpg"},
},
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Value(),
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.Uploaded, 4).Value(),
},
{
name: "date on name",
Expand All @@ -67,7 +69,7 @@ func TestE2ELocalAssets(t *testing.T) {
expectedFiles: map[string]fileLinks{
"photo1_2023-10-06_wo_exif.jpg": {image: "photo1_2023-10-06_wo_exif.jpg"},
},
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 3).Value(),
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 3).Set(fileevent.Uploaded, 1).Value(),
},
{
name: "select exif date not using exiftool",
Expand All @@ -94,7 +96,7 @@ func TestE2ELocalAssets(t *testing.T) {
"photo1_w_exif.jpg": {image: "photo1_w_exif.jpg"},
"photo1_2024-10-06_w_exif.jpg": {image: "photo1_2024-10-06_w_exif.jpg"},
},
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 2).Value(),
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 2).Set(fileevent.Uploaded, 2).Value(),
},
{
name: "select exif date using exiftool",
Expand All @@ -121,7 +123,7 @@ func TestE2ELocalAssets(t *testing.T) {
"photo1_w_exif.jpg": {image: "photo1_w_exif.jpg"},
"photo1_2024-10-06_w_exif.jpg": {image: "photo1_2024-10-06_w_exif.jpg"},
},
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 2).Value(),
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 2).Set(fileevent.Uploaded, 2).Value(),
},
{
name: "select exif date using exiftool then date",
Expand Down Expand Up @@ -149,7 +151,7 @@ func TestE2ELocalAssets(t *testing.T) {
"photo1_2023-10-06_wo_exif.jpg": {image: "photo1_2023-10-06_wo_exif.jpg"},
"photo1_2024-10-06_w_exif.jpg": {image: "photo1_2024-10-06_w_exif.jpg"},
},
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 1).Value(),
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 1).Set(fileevent.Uploaded, 3).Value(),
},
{
name: "select on date in the name",
Expand All @@ -175,7 +177,7 @@ func TestE2ELocalAssets(t *testing.T) {
expectedFiles: map[string]fileLinks{
"photo1_2023-10-06_wo_exif.jpg": {image: "photo1_2023-10-06_wo_exif.jpg"},
},
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 3).Value(),
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 4).Set(fileevent.DiscoveredDiscarded, 3).Set(fileevent.Uploaded, 1).Value(),
},
{
name: "same name, but not live photo, select exif date using exiftool then date",
Expand All @@ -202,14 +204,26 @@ func TestE2ELocalAssets(t *testing.T) {
"IMG_1234.jpg": {image: "IMG_1234.jpg"},
"IMG_1234.mp4": {video: "IMG_1234.mp4"},
},
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 1).Set(fileevent.DiscoveredVideo, 1).Value(),
expectedCounts: fileevent.NewCounts().Set(fileevent.DiscoveredImage, 1).Set(fileevent.DiscoveredVideo, 1).Set(fileevent.Uploaded, 2).Value(),
},
}

logFile := configuration.DefaultLogFile()
for _, c := range tc {
t.Run(c.name, func(t *testing.T) {
ctx := context.Background()
recorder := fileevent.NewRecorder(nil)

log := application.Log{
File: logFile,
Level: "INFO",
}
err := log.OpenLogFile()
if err != nil {
t.Error(err)
return
}
log.Info("Test case: " + c.name)
recorder := fileevent.NewRecorder(log.Logger)
b, err := NewLocalFiles(ctx, recorder, &c.flags, c.fsys...)
if err != nil {
t.Error(err)
Expand Down Expand Up @@ -239,6 +253,7 @@ func TestE2ELocalAssets(t *testing.T) {
links.video = a.FileName
}
a.Close()
recorder.Record(ctx, fileevent.Uploaded, a)
}
results[fileName] = links
if len(c.expectedAlbums) > 0 {
Expand Down
2 changes: 1 addition & 1 deletion adapters/folder/readFolder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func (mfs *inMemFS) addFile(name string) *inMemFS {
return mfs
}

func TestLocalAssets(t *testing.T) {
func TestInMemLocalAssets(t *testing.T) {
tc := []struct {
name string
fsys []fs.FS
Expand Down
8 changes: 4 additions & 4 deletions adapters/googlePhotos/e2etests/e2e_nonregression_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ func initMyEnv(t *testing.T) {
t.Fatalf("cant initialize environment variables: %s", err)
}
myEnv = e
if myEnv["IMMICH_TESTFILES"] == "" {
t.Fatal("missing IMMICH_TESTFILES in .env file")
if myEnv["IMMICHGO_TESTFILES"] == "" {
t.Fatal("missing IMMICHGO_TESTFILES in .env file")
}
}

Expand Down Expand Up @@ -91,10 +91,10 @@ func simulate_upload(testname string, flags *gp.ImportFlags, fsys []fs.FS) (*fil
}
for g := range assetsGroups {
for i, a := range g.Assets {
jnl.Record(ctx, fileevent.Uploaded, fileevent.AsFileAndName(a.FSys, a.FileName))
jnl.Record(ctx, fileevent.Uploaded, a)
if i >= 0 {
for _, album := range g.Albums {
jnl.Record(ctx, fileevent.UploadAddToAlbum, fileevent.AsFileAndName(a.FSys, a.FileName), "album", album.Title)
jnl.Record(ctx, fileevent.UploadAddToAlbum, a, "album", album.Title)
}
}
}
Expand Down
14 changes: 7 additions & 7 deletions adapters/googlePhotos/e2etests/e2e_takeout_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
func TestPixilTakeOut(t *testing.T) {
initMyEnv(t)

files := myEnv["IMMICH_TESTFILES"] + "/User Files/pixil/0.22.0/list.lst.zip"
files := myEnv["IMMICHGO_TESTFILES"] + "/User Files/pixil/0.22.0/list.lst.zip"
fsyss, err := fakefs.ScanFileList(files, "01-02-2006 15:04")
if err != nil {
t.Error(err)
Expand All @@ -40,7 +40,7 @@ func TestPixilTakeOut(t *testing.T) {
func TestDemoTakeOut(t *testing.T) {
initMyEnv(t)

files := myEnv["IMMICH_TESTFILES"] + "/demo takeout/Takeout"
files := myEnv["IMMICHGO_TESTFILES"] + "/demo takeout/Takeout"
fsyss, err := fshelper.ParsePath([]string{files})
if err != nil {
t.Error(err)
Expand Down Expand Up @@ -72,7 +72,7 @@ In this dataset, a file can be present in different ZIP files, with the same pat
func TestPhyl404TakeOut(t *testing.T) {
initMyEnv(t)

files := myEnv["IMMICH_TESTFILES"] + "/User Files/Phyl404/list.lst"
files := myEnv["IMMICHGO_TESTFILES"] + "/User Files/Phyl404/list.lst"
fsyss, err := fakefs.ScanFileList(files, "2006-01-02 15:04")
if err != nil {
t.Error(err)
Expand All @@ -97,7 +97,7 @@ func TestPhyl404TakeOut(t *testing.T) {
func TestPhyl404_2TakeOut(t *testing.T) {
initMyEnv(t)

files := myEnv["IMMICH_TESTFILES"] + "/User Files/Phyl404#2/list.lst"
files := myEnv["IMMICHGO_TESTFILES"] + "/User Files/Phyl404#2/list.lst"
fsyss, err := fakefs.ScanFileList(files, "2006-01-02 15:04")
if err != nil {
t.Error(err)
Expand All @@ -121,7 +121,7 @@ func TestPhyl404_2TakeOut(t *testing.T) {
func TestSteve81TakeOut(t *testing.T) {
initMyEnv(t)

files := myEnv["IMMICH_TESTFILES"] + "/User Files/Steve81/list.list"
files := myEnv["IMMICHGO_TESTFILES"] + "/User Files/Steve81/list.list"
fsyss, err := fakefs.ScanFileList(files, "2006-01-02 15:04")
if err != nil {
t.Error(err)
Expand All @@ -146,7 +146,7 @@ func TestSteve81TakeOut(t *testing.T) {
func TestMuetyTakeOut(t *testing.T) {
initMyEnv(t)

files := myEnv["IMMICH_TESTFILES"] + "/User Files/muety/list.lst.zip"
files := myEnv["IMMICHGO_TESTFILES"] + "/User Files/muety/list.lst.zip"
fsyss, err := fakefs.ScanFileList(files, "01-02-2006 15:04")
if err != nil {
t.Error(err)
Expand All @@ -172,7 +172,7 @@ func TestMuetyTakeOut(t *testing.T) {
func TestMissingJSONTakeOut(t *testing.T) {
initMyEnv(t)

files := myEnv["IMMICH_TESTFILES"] + "/User Files/MissingJSON/list.lst"
files := myEnv["IMMICHGO_TESTFILES"] + "/User Files/MissingJSON/list.lst"
fsyss, err := fakefs.ScanFileList(files, "01-02-2006 15:04")
if err != nil {
t.Error(err)
Expand Down
Loading

0 comments on commit 0bf90c2

Please sign in to comment.