Added chunk mode for dealing with really big file

axgarg3 · Aug 10, 2020 · 0a26473 · 0a26473
1 parent 5b81234
commit 0a26473
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/cmd/root.go b/cmd/root.go
@@ -86,7 +86,7 @@ func init() {
 	RootCmd.PersistentFlags().BoolVar(&options.Debug, "debug", false, "Debug")
 	// chunk options
 	RootCmd.PersistentFlags().BoolVar(&options.ChunkRun, "chunk", false, "Enable chunk running against big input")
-	RootCmd.PersistentFlags().IntVarP(&options.Threads, "threads","t", 1, "Number of Threads")
+	RootCmd.PersistentFlags().IntVarP(&options.Threads, "threads","t", 1, "Number of Threads (Only used in chunk mode)")
 	RootCmd.PersistentFlags().IntVar(&options.ChunkSize, "chunk-size", 10000, "Chunk Size")
 	RootCmd.PersistentFlags().StringVar(&options.ChunkDir, "chunk-dir", "", "Temp Directory to store chunk directory")
 	RootCmd.PersistentFlags().IntVar(&options.ChunkLimit, "chunk-limit", 500000, "Limit size to trigger chunk run")

diff --git a/cmd/scan.go b/cmd/scan.go
@@ -523,14 +523,14 @@ func runChunk(command string, urlFiles []string, threads int) {
 
 	var wg sync.WaitGroup
 	p, _ := ants.NewPoolWithFunc(threads, func(i interface{}) {
-		defer wg.Done()
 		cmd := i.(string)
 		ExecutionWithStd(cmd)
+		wg.Done()
 	}, ants.WithPreAlloc(true))
 	defer p.Release()
 	for _, command := range commands {
 		wg.Add(1)
-		p.Invoke(command)
+		_ = p.Invoke(command)
 	}
 	wg.Wait()
 }