added Allow to create custom settings for scraper-config.json #139

README.md

@@ -95,6 +95,7 @@ $ cloud-torrent --help
   --host, -h         Listening interface (default all)
   --auth, -a         Optional basic auth in form 'user:password' (env AUTH)
   --config-path, -c  Configuration file path (default cloud-torrent.json)
+  --config           Search config file path
   --key-path, -k     TLS Key file path
   --cert-path, -r    TLS Certicate file path
   --log, -l          Enable request logging

scraper-config.json

@@ -0,0 +1,110 @@
+{
+  "eztv": {
+    "name": "EZTV",
+    "url": "https://eztv.ag/search/{{query}}",
+    "list": "table tr.forum_header_border",
+    "result": {
+      "name": "td:nth-child(2) a",
+      "url": ["td:nth-child(2) a", "@href"],
+      "magnet": ["td:nth-child(3) a:nth-child(1)", "@href"],
+      "size": "td:nth-child(4)",
+      "seeds": "td:nth-child(6)"
+    }
+  },
+  "1337x": {
+    "name": "1337X",
+    "url": "http://1337x.to/sort-search/{{query}}/seeders/desc/{{page:1}}/",
+    "list": ".box-info-detail table.table tr",
+    "result": {
+      "name": [".coll-1 a:nth-child(2)"],
+      "url": [".coll-1 a:nth-child(2)", "@href"],
+      "seeds": ".coll-2",
+      "peers": ".coll-3",
+      "size": [".coll-4", "/([\\d\\.]+ [KMGT]?B)/"]
+    }
+  },
+  "1337x/item": {
+    "name": "1337X (Item)",
+    "url": "http://1337x.to{{item}}",
+    "result": {
+      "magnet": ["a[href^='magnet:']", "@href"]
+    }
+  },
+  "abb": {
+    "name": "The Audiobook Bay",
+    "url": "http://audiobookbay.nl/page/{{page:1}}?s={{query}}",
+    "list": "#content > .post",
+    "result": {
+      "name": ["div.postTitle > h2 > a"],
+      "url": ["div.postTitle > h2 > a", "@href"],
+      "size": ["div.postContent", "/File Size: ([\\d\\.]+ [KMGT]?B)/"]
+    }
+  },
+  "abb/item": {
+    "name": "The Audiobook Bay (Item)",
+    "url": "http://audiobookbay.nl{{item}}",
+    "result": {
+      "infohash": "/td>([a-f0-9]+)</",
+      "tracker": "table.torrent_info tr:nth-child(1) td:nth-child(2)"
+    }
+  },
+  "lt": {
+    "name": "limetorrents",
+    "url": "https://www.limetorrents.cc/search/all/{{query}}/seeds/{{page:1}}/",
+    "list": ".table2 tbody tr",
+    "result": {
+      "name": [".tt-name a:nth-child(2)"],
+      "url": [".tt-name a:nth-child(2)", "@href"],
+      "size": "td:nth-child(3)",
+      "seeds": "td:nth-child(4)",
+      "peers": "td:nth-child(5)"
+    }
+  },
+  "lt/item": {
+    "name": "limetorrents (Item)",
+    "url": "https://www.limetorrents.cc{{item}}",
+    "result": {
+      "magnet": ["a.csprite_dltorrent[href^=\"magnet:\"]", "@href"]
+    }
+  },
+  "tpb": {
+    "name": "The Pirate Bay",
+    "url": "https://thepiratebay.org/search/{{query}}/{{page:0}}/7//",
+    "list": "#searchResult > tbody > tr",
+    "result": {
+      "name": "a.detLink",
+      "path": ["a.detLink", "@href"],
+      "magnet": [
+        "a[title=Download\\ this\\ torrent\\ using\\ magnet]",
+        "@href"
+      ],
+      "size": "/Size (\\d+(\\.\\d+).[KMG]iB)/",
+      "seeds": "td:nth-child(3)",
+      "peers": "td:nth-child(4)"
+    }
+  },
+  "nyaa": {
+    "name": "Nyaa",
+    "url": "https://nyaa.si/?f=0&c=0_0&s=seeders&o=desc&q={{query}}&p={{page:1}}",
+    "list": "table.torrent-list > tbody > tr",
+    "result": {
+      "name": ["td:nth-child(2) a:nth-child(2)", "@title"],
+      "path": ["td:nth-child(2) a:nth-child(2)", "@href"],
+      "magnet": ["a[href^='magnet:']", "@href"],
+      "size": "td:nth-child(4)",
+      "seeds": "td:nth-child(6)",
+      "peers": "td:nth-child(7)"
+    }
+  },
+  "zhangyusousuo": {
+    "name": "章鱼搜索",
+    "url": "https://bt.zhangyusousuo.com/s/{{query}}/p/{{page:1}}.html",
+    "list": ".listitem:nth-child(n+3)",
+    "result": {
+      "name": "h2 > a",
+      "url": ["h2 > a", "@href"],
+      "size": ["/大小:(.*)</","s/ //"],
+      "magnet": ["/hash: (.*)\t/","s/^/magnet:?xt=urn:btih:/"]
+    }
+  }
+}

server/server.go

@@ -33,6 +33,7 @@ type Server struct {
 	Host       string `help:"Listening interface (default all)"`
 	Auth       string `help:"Optional basic auth in form 'user:password'" env:"AUTH"`
 	ConfigPath string `help:"Configuration file path"`
+	Config     string `help:"Search config file path"`
 	KeyPath    string `help:"TLS Key file path"`
 	CertPath   string `help:"TLS Certicate file path" short:"r"`
 	Log        bool   `help:"Enable request logging"`
@@ -89,7 +90,9 @@ func (s *Server) Run(version string) error {
 	}
 	//scraper
 	s.state.SearchProviders = s.scraper.Config //share scraper config
-	go s.fetchSearchConfigLoop()
+	if err:=s.fetchSearchConfigOffFile(); err!=nil {
+		go s.fetchSearchConfigLoop()
+	}
 	s.scraperh = http.StripPrefix("/search", s.scraper)
 	//torrent engine
 	s.engine = engine.New()

server/server_search.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/jpillora/backoff"
+	"os"
 )
 
 const searchConfigURL = "https://gist.githubusercontent.com/jpillora/4d945b46b3025843b066adf3d685be6b/raw/scraper-config.json"
@@ -30,6 +31,34 @@ func (s *Server) fetchSearchConfigLoop() {
 var fetches = 0
 var currentConfig, _ = normalize(defaultSearchConfig)
 
+func (s *Server) fetchSearchConfigOffFile() error {
+	f,err := os.Open(s.Config)
+	defer f.Close()
+	if err != nil {
+		return err
+	}
+	newConfig,err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
+	newConfig, err = normalize(newConfig)
+	if err != nil {
+		return err
+	}
+	fetches++
+	if bytes.Equal(currentConfig, newConfig) {
+		return nil //skip
+	}
+	if err := s.scraper.LoadConfig(newConfig); err != nil {
+		return err
+	}
+	s.state.SearchProviders = s.scraper.Config
+	s.state.Push()
+	currentConfig = newConfig
+	log.Printf("Loaded new search providers")
+	return nil
+}
+
 func (s *Server) fetchSearchConfig() error {
 	resp, err := http.Get(searchConfigURL)
 	if err != nil {