Skip to content

Commit

Permalink
Fix external sources (#88)
Browse files Browse the repository at this point in the history
* Replace partial ASN with official stackpath IP list

Based on https://support.stackpath.com/hc/en-us/articles/360001091666-Review-and-Allowlist-CDN-WAF-IP-Blocks

* Fix download of Azure IPs and tweak logic not to break for stackpath or other non-JSON sources

* Remove my debug code and clarify comment
  • Loading branch information
neonbunny authored May 27, 2023
1 parent bd38997 commit fe17530
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 8 deletions.
8 changes: 4 additions & 4 deletions cmd/generate-index/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ cdn:
asn:
leaseweb:
- AS60626
stackpath:
- AS64259

# urls contains a list of URLs for CDN providers
urls:
Expand All @@ -18,6 +16,8 @@ cdn:
- https://api.fastly.com/public-ip-list
google:
- https://www.gstatic.com/ipranges/goog.json
stackpath:
- https://k3t9x2h3.map2.ssl.hwcdn.net/ipblocks.txt

# waf contains the inputs for WAF CIDR checking
waf:
Expand All @@ -44,7 +44,7 @@ cloud:
oracle:
- https://docs.oracle.com/en-us/iaas/tools/public_ip_ranges.json
azure:
- https://download.microsoft.com/download/7/1/D/71D86715-5596-4529-9B13-DA13A5DE5B63/ServiceTags_Public_20221128.json
- https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519
zscaler:
- https://api.config.zscaler.com/zscaler.net/cenr/json
office365:
Expand All @@ -69,4 +69,4 @@ common:
edgecast:
- edgecastcdn.net
incapsula:
- impervadns.net
- impervadns.net
6 changes: 2 additions & 4 deletions generate/input.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package generate

import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -144,9 +143,8 @@ retry:
if err != nil {
return nil, err
}
// if the body type is not json retry with the first json link in the page
unmarshaledData := make(map[string]interface{})
if err := json.Unmarshal(data, &unmarshaledData); err != nil && !retried {
// if the body type is HTML, retry with the first json link in the page (special case for Azure download page to find changing URLs)
if resp.Header.Get("Content-Type") == "text/html" && !retried {
var extractedURL string
docReader, err := goquery.NewDocumentFromReader(bytes.NewReader(data))
if err != nil {
Expand Down

0 comments on commit fe17530

Please sign in to comment.