Merge pull request #3 from twiny/dev

code clean

Author: twiny

README.md

@@ -14,6 +14,28 @@ go build -o bin/spidy -v cmd/spidy/main.go
 ./bin/spidy -c config/config.yaml -u https://github.com
 ```
 
+## Usage
+
+```sh
+NAME:
+   Spidy - Domain name scraper
+
+USAGE:
+   spidy [global options] command [command options] [arguments...]
+
+VERSION:
+   2.0.0
+
+COMMANDS:
+   help, h  Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --config path, -c path  path to config file
+   --help, -h              show help (default: false)
+   --urls urls, -u urls    urls of page to scrape  (accepts multiple inputs)
+   --version, -v           print the version (default: false)
+```
+
 ## Configuration
 
 ```yaml

cmd/spidy/api/spider.go

@@ -13,7 +13,6 @@ import (
 	"syscall"
 
 	//
-
 	"github.com/twiny/spidy/v2/internal/pkg/spider/v1"
 	"github.com/twiny/spidy/v2/internal/service/cache"
 	"github.com/twiny/spidy/v2/internal/service/writer"

internal/pkg/spider/v1/setting.go

@@ -45,7 +45,7 @@ var defaultSetting = &Setting{
 			Interval: time.Second,
 		},
 		MaxBodySize: 10 * 1024 * 1024, // 10 MB
-		UserAgents:  []string{`Spidy v2 - Agent.`},
+		UserAgents:  []string{`Spidy/2.1; +https://github.com/twiny/spidy`},
 		Proxies:     []string{},
 	},
 	Log: struct {