Skip to content

Commit

Permalink
Add support for custom HTTP headers in feed requests and update README
Browse files Browse the repository at this point in the history
  • Loading branch information
TBosak committed Nov 22, 2024
1 parent 048af99 commit a542a02
Show file tree
Hide file tree
Showing 4 changed files with 7 additions and 4 deletions.
3 changes: 1 addition & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ Access the GUI at `http://localhost:5000/`
- [ ] Utilities
- [X] HTML stripper
- [X] Source url wrapper for relative links
- [ ] ...
- [ ] Amass contributors (the more, the merrier)
- [ ] Amass contributors

[![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=V5LC4XTQDDE82&source=url)
4 changes: 3 additions & 1 deletion index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,13 +175,15 @@ app.post('/', async (ctx) => {
false,
extractValue("dateIterator")
);
const headers = extractValue("headers");

article = {
iterator: iteratorTarget,
title: titleTarget,
description: descriptionTarget,
link: linkTarget,
date: dateTarget,
headers: headers,
}
}
else if (feedType === 'api') {
Expand Down Expand Up @@ -583,7 +585,7 @@ async function generatePreview(feedConfig: any) {
var rssXml;

if(feedConfig.feedType === 'webScraping') {
const response = await axios.get(feedConfig.config.baseUrl);
const response = feedConfig.article.headers ? await axios.get(feedConfig.config.baseUrl, {headers: feedConfig.article.headers}) : await axios.get(feedConfig.config.baseUrl);
const html = response.data;
// Generate the RSS feed using your buildRSS function
rssXml = buildRSS(
Expand Down
2 changes: 2 additions & 0 deletions public/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ <h3>API Response Mapping</h3>
<br><br>
<details>
<summary>Additional Options</summary>
<label for="headers">HTTP Headers (JSON format):</label>
<textarea name="headers" id="headers" rows="5" cols="50">{}</textarea>
<label for="refreshTime">Refresh Time (in minutes):</label>
<input type="text" name="refreshTime" id="refreshTime" min="1" value="5">
<label>
Expand Down
2 changes: 1 addition & 1 deletion workers/feed-updater.worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ async function fetchDataAndUpdateFeed(feedConfig) {
var rssXml;

if(feedConfig.feedType === 'webScraping') {
const response = await axios.get(feedConfig.config.baseUrl);
const response = feedConfig.article.headers ? await axios.get(feedConfig.config.baseUrl, {headers: feedConfig.article.headers}) : await axios.get(feedConfig.config.baseUrl);
const html = response.data;
// Generate the RSS feed using your buildRSS function
rssXml = buildRSS(
Expand Down

0 comments on commit a542a02

Please sign in to comment.