fix: scraping not working

Author: Bash06

api/share.go

@@ -18,21 +18,14 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 package api
 
 import (
-	"bash06/vxinstagram/flags"
-	"bash06/vxinstagram/utils"
-	"log/slog"
 	"bash06/vxinstagram/flags"
 	"bash06/vxinstagram/utils"
 	"log/slog"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"strings"
-	"net/http/httputil"
-	"net/url"
-	"strings"
 
-	"github.com/getsentry/sentry-go"
 	"github.com/getsentry/sentry-go"
 	"github.com/gin-gonic/gin"
 )
@@ -64,17 +57,6 @@ func FollowShare(c *gin.Context) {
 		return
 	}
 
-	res, err := client.Do(req)
-	if err != nil {
-		slog.Error("Failed to follow redirects", slog.Any("err", err))
-		sentry.CaptureException(err)
-		c.HTML(http.StatusOK, "embed.html", &HtmlOpenGraphData{
-			Title:       "VxInstagram - Server Error",
-			Description: "VxInstagram encountered a server side error while processing your request. Request ID:`" + span.SpanID.String() + "`",
-		})
-		return
-	}
-	res.Body.Close()
 	res, err := client.Do(req)
 	if err != nil {
 		slog.Error("Failed to follow redirects", slog.Any("err", err))
@@ -87,8 +69,6 @@ func FollowShare(c *gin.Context) {
 	}
 	res.Body.Close()
 
-	urlSplit := strings.Split(res.Request.URL.String(), "/")
-	postId := urlSplit[len(urlSplit)-2]
 	urlSplit := strings.Split(res.Request.URL.String(), "/")
 	postId := urlSplit[len(urlSplit)-2]
 
@@ -130,16 +110,6 @@ func FollowShare(c *gin.Context) {
 			})
 		}
 	}
-		videoUrl = data.Items[0].VideoVersions[0].URL
-
-		if videoUrl == "" {
-			slog.Debug("No video URL found! :(")
-			c.HTML(http.StatusOK, "embed.html", &HtmlOpenGraphData{
-				Title:       "VxInstagram - Empty Response",
-				Description: "Instagram returned an empty response meaning we can't embed the video. You'll need to watch it in your browser. Sorry!",
-			})
-		}
-	}
 
 	remote, err := url.Parse(videoUrl)
 	if err != nil {
@@ -150,24 +120,9 @@ func FollowShare(c *gin.Context) {
 			Description: "VxInstagram encountered a server side error while processing your request. Request ID:`" + span.SpanID.String() + "`",
 		})
 		return
-	}
-	remote, err := url.Parse(videoUrl)
-	if err != nil {
-		slog.Error("Failed to parse CDN video URL", slog.Any("err", err))
-		sentry.CaptureException(err)
-		c.HTML(http.StatusOK, "embed.html", &HtmlOpenGraphData{
-			Title:       "VxInstagram - Server Error",
-			Description: "VxInstagram encountered a server side error while processing your request. Request ID:`" + span.SpanID.String() + "`",
-		})
-		return
+
 	}
 
-	proxy := httputil.NewSingleHostReverseProxy(remote)
-	proxy.Director = func(r *http.Request) {
-		r.Header = c.Request.Header
-		r.Host = remote.Host
-		r.URL = remote
-		r.Header = c.Request.Header.Clone()
 	proxy := httputil.NewSingleHostReverseProxy(remote)
 	proxy.Director = func(r *http.Request) {
 		r.Header = c.Request.Header
@@ -178,14 +133,6 @@ func FollowShare(c *gin.Context) {
 		hopHeaders := []string{
 			"Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "Te", "Trailer", "Transfer-Encoding",
 		}
-		hopHeaders := []string{
-			"Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "Te", "Trailer", "Transfer-Encoding",
-		}
-
-		for _, h := range hopHeaders {
-			r.Header.Del(h)
-		}
-	}
 		for _, h := range hopHeaders {
 			r.Header.Del(h)
 		}
@@ -194,7 +141,4 @@ func FollowShare(c *gin.Context) {
 	slog.Debug("Success!")
 	c.Header("Cache-Control", "max-age=43200")
 	proxy.ServeHTTP(c.Writer, c.Request)
-	slog.Debug("Success!")
-	c.Header("Cache-Control", "max-age=43200")
-	proxy.ServeHTTP(c.Writer, c.Request)
 }

utils/extractUrl.go

@@ -17,7 +17,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package utils
 
-import "strings"
+import (
+	"fmt"
+	"strings"
+)
 
 const (
 	prefix    = `\"video_url\":`
@@ -50,5 +53,7 @@ func ExtractUrl(s string) (string, bool) {
 	result = UnescapeJSONString(result)
 	result = strings.ReplaceAll(result, `\/`, `/`)
 
+	fmt.Println(result)
+
 	return result[1:], true
 }

utils/fetch.go

@@ -21,7 +21,6 @@ import (
 	"bash06/vxinstagram/flags"
 	"fmt"
 	"net/http"
-	"net/url"
 
 	jsoniter "github.com/json-iterator/go"
 )
@@ -74,25 +73,19 @@ func FetchPost(postId string) (*IgResponse, error) {
 
 	baseURL := "https://www.instagram.com/p/" + postId + "?__a=1&__d=dis"
 
-	url, err := url.Parse(baseURL)
-	if err != nil {
-		return nil, err
-	}
-
-	q := url.Query()
-	q.Set("__a", "1")
-	q.Set("__d", "dis")
-	url.RawQuery = q.Encode()
-
-	req, err := http.NewRequest("GET", url.String(), nil)
+	req, err := http.NewRequest("GET", baseURL, nil)
 	if err != nil {
 		return nil, err
 	}
 
 	req.Header.Set("User-Agent", *flags.InstagramBrowserAgent)
 	req.Header.Set("Cookie", *flags.InstagramCookie)
 	req.Header.Set("X-IG-App-ID", *flags.InstagramXIGAppID)
+
+	// Set headers so we look more like a real browser
 	req.Header.Set("Sec-Fetch-Site", "same-origin")
+	req.Header.Set("Origin", "https://www.instagram.com")
+	req.Header.Set("Referer", "https://www.instagram.com")
 
 	resp, err := GetIpRotationClient(5).Do(req)
 	if err != nil {

utils/scrape.go

@@ -35,7 +35,7 @@ func ScrapeFromHTML(postId string) (string, error) {
 		return "", fmt.Errorf("failed to prepare HTTP request: %v", err)
 	}
 
-	req.Header.Set("User-Agent", *flags.InstagramBrowserAgent)
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
 
 	var client *http.Client