Skip to content

Commit 74b64df

Browse files
MeNsaaHMeNsaaH
committed
Added CloudFlare Scraping fix to NetNaija
1 parent ca517df commit 74b64df

File tree

4 files changed

+31
-0
lines changed

4 files changed

+31
-0
lines changed

‎engine/engines.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@ import (
44
"encoding/json"
55
"errors"
66
"fmt"
7+
"net/http"
78
"net/url"
89
"strconv"
910
"strings"
1011

1112
"github.com/gocolly/colly/v2"
13+
scraper "github.com/mensaah/go-cloudflare-scraper"
1214
log "github.com/sirupsen/logrus"
1315
)
1416

@@ -28,6 +30,7 @@ func (m Mode) String() string {
2830

2931
// Engine : interface for all engines
3032
type Engine interface {
33+
getName() string
3134
getParseURL() *url.URL
3235
Search(param ...string) SearchResult
3336
List(page int) SearchResult
@@ -52,6 +55,16 @@ func Scrape(engine Engine) ([]Movie, error) {
5255
// even if the collector is restarted
5356
colly.CacheDir("./gophie_cache"),
5457
)
58+
59+
// Add Cloud Flare scraper bypasser
60+
if engine.getName() == "netnaija" {
61+
t, err := scraper.NewTransport(http.DefaultTransport)
62+
if err != nil {
63+
log.Fatal(err)
64+
}
65+
c.WithTransport(t)
66+
}
67+
5568
// Another collector for download Links
5669
downloadLinkCollector := c.Clone()
5770

‎engine/props.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,7 @@ func (p *Props) getParseURL() *url.URL {
4242
}
4343
return p.ListURL
4444
}
45+
46+
func (p *Props) getName() string {
47+
return p.Name
48+
}

‎go.mod‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,13 @@ require (
1616
github.com/golang/protobuf v1.3.4 // indirect
1717
github.com/iawia002/annie v0.0.0-20200217104547-c4b096ad402a
1818
github.com/manifoldco/promptui v0.7.0
19+
github.com/mensaah/go-cloudflare-scraper v0.0.0-20200516233131-970abdc95990
1920
github.com/mitchellh/go-homedir v1.1.0
21+
github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff // indirect
2022
github.com/sirupsen/logrus v1.4.2
2123
github.com/spf13/cobra v0.0.5
2224
github.com/spf13/viper v1.6.2
2325
golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect
2426
google.golang.org/appengine v1.6.5 // indirect
27+
gopkg.in/sourcemap.v1 v1.0.5 // indirect
2528
)

‎go.sum‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ github.com/bisoncorps/mplayer v0.0.0-20200330192254-e2f647162350 h1:562G6+GDTx5K
4141
github.com/bisoncorps/mplayer v0.0.0-20200330192254-e2f647162350/go.mod h1:KT1eUGk8Y07NyBhjTrKe0r0eqhv7DQRlJpLLii1kH8s=
4242
github.com/briandowns/spinner v1.9.0 h1:+OMAisemaHar1hjuJ3Z2hIvNhQl9Y7GLPWUwwz2Pxo8=
4343
github.com/briandowns/spinner v1.9.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ=
44+
github.com/cardigann/go-cloudflare-scraper v0.0.0-20200425223932-91bd9b1006f2 h1:DYQ/ugkvCFdvZUf5d6RlLxmZTL3R6NOAjfMcCXZBmEA=
45+
github.com/cardigann/go-cloudflare-scraper v0.0.0-20200425223932-91bd9b1006f2/go.mod h1:yzwU7pub3vYvHwvYOpbXfJ5OC8srFoqFmij8ZtQpymI=
4446
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
4547
github.com/cheggaaa/pb v1.0.25 h1:tFpebHTkI7QZx1q1rWGOKhbunhZ3fMaxTvHDWn1bH/4=
4648
github.com/cheggaaa/pb v1.0.25/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
@@ -149,6 +151,10 @@ github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
149151
github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
150152
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
151153
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
154+
github.com/mensaah/go-cloudflare-scraper v0.0.0-20200425223932-91bd9b1006f2 h1:brtTpjMbnjJZ0mI5oY/ClawRRFMxg4jlj/iBFzhl4CA=
155+
github.com/mensaah/go-cloudflare-scraper v0.0.0-20200425223932-91bd9b1006f2/go.mod h1:0YYv3nDDuXyAYCaTVtEP+opfkpCkBRe0pROWyZ6twyI=
156+
github.com/mensaah/go-cloudflare-scraper v0.0.0-20200516233131-970abdc95990 h1:mjQJJSQdejeXemoU0m3wg5to5qzyi0yKbiz7kOC3FxU=
157+
github.com/mensaah/go-cloudflare-scraper v0.0.0-20200516233131-970abdc95990/go.mod h1:1ctjBua3T8LD8mbkJM3CQMMrRS5rASJ64bepnch4d9w=
152158
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
153159
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
154160
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
@@ -171,6 +177,9 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
171177
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
172178
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
173179
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
180+
github.com/robertkrimen/otto v0.0.0-20170604103602-49fee9c1c0db/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY=
181+
github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff h1:+6NUiITWwE5q1KO6SAfUX918c+Tab0+tGAM/mtdlUyA=
182+
github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY=
174183
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
175184
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
176185
github.com/rs/zerolog v1.16.0 h1:AaELmZdcJHT8m6oZ5py4213cdFK8XGXkB3dFdAQ+P7Q=
@@ -308,6 +317,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
308317
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
309318
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
310319
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
320+
gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
321+
gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
311322
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
312323
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
313324
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

0 commit comments

Comments
 (0)