fix: remove hard-coded file limit

trustification · Jan 31, 2024 · cfb9987 · cfb9987
1 parent f03b736
commit cfb9987
Showing 1 changed file with 5 additions and 4 deletions.
diff --git a/pkg/handler/collector/s3/s3.go b/pkg/handler/collector/s3/s3.go
@@ -124,10 +124,11 @@ func retrieve(s S3Collector, ctx context.Context, docChannel chan<- *processor.D
 					continue
 				}
 
-				if len(blob) > 6291456 {
-					logger.Infof("Skipping %s due to its size %d", item, len(blob))
-					continue
-				}
+				// TODO make this configurable
+				// if len(blob) > 6291456 {
+				// 	logger.Infof("Skipping %s due to its size %d", item, len(blob))
+				// 	continue
+				// }
 
 				enc, err := downloader.GetEncoding(ctx, s.config.S3Bucket, item)
 				if err != nil {