Added crawler, feed support, filter and sorting

.env

@@ -0,0 +1 @@
+NODE_ENV=dev

package.json

@@ -1,98 +1,107 @@
 {
-  "name": "sc2-campaign-manager",
-  "version": "0.5.3-alpha",
-  "description": "A Campaign Manager for StarCraft II.  ",
-  "main": "main.js",
-  "build": {
-    "appId": "com.sc2mapster.campaignmanager",
-    "mac": {
-      "category": "demo.app"
-    },
-    "win": {
-      "target": "nsis"
-    }
-  },
-  "scripts": {
-    "dev": "cross-env NODE_ENV=dev webpack --watch",
-    "start": "nf start",
-    "webpack": "webpack",
-    "electron": "cross-env NODE_ENV=dev electron .",
-    "build": "cross-env NODE_ENV=prod electron-builder"
-  },
-  "repository": "https://github.com/abrahamYG/sc2-campaign-manager",
-  "keywords": [
-    "Starcraft",
-    "demo"
-  ],
-  "author": "abrahamYG",
-  "license": "CC0-1.0",
-  "devDependencies": {
-    "@babel/cli": "^7.2.3",
-    "@babel/core": "^7.4.3",
-    "@babel/plugin-proposal-class-properties": "^7.4.0",
-    "@babel/preset-env": "^7.3.1",
-    "@babel/preset-react": "^7.0.0",
-    "@babel/register": "^7.4.0",
-    "@fortawesome/fontawesome-free": "^5.8.1",
-    "awesome-typescript-loader": "^5.2.1",
-    "babel": "^6.23.0",
-    "babel-cli": "^6.26.0",
-    "babel-loader": "^8.0.5",
-    "babel-preset-flow": "^6.23.0",
-    "cross-env": "^5.2.0",
-    "css-loader": "^2.1.0",
-    "electron": "^4.0.4",
-    "electron-builder": "^20.41.0",
-    "electron-devtools-installer": "^2.2.4",
-    "font-awesome-loader": "^1.0.2",
-    "foreman": "^3.0.1",
-    "react-hot-loader": "^4.7.0",
-    "sass-loader": "^7.1.0",
-    "source-map-loader": "^0.2.4",
-    "style-loader": "^0.23.1",
-    "typescript": "^3.4.2",
-    "webpack": "^4.29.5",
-    "webpack-cli": "^3.2.3",
-    "webpack-dev-server": "^3.1.14"
-  },
-  "dependencies": {
-    "@types/debug": "^4.1.3",
-    "@types/download": "^6.2.4",
-    "@types/electron": "^1.6.10",
-    "@types/fs-extra": "^7.0.0",
-    "@types/lodash": "^4.14.130",
-    "@types/react": "^16.8.13",
-    "@types/react-dom": "^16.8.3",
-    "@types/react-images": "^0.5.1",
-    "@types/react-jsonschema-form": "^1.3.2",
-    "@types/react-redux": "^7.0.9",
-    "@types/react-router-dom": "^4.3.1",
-    "@types/yauzl": "^2.9.1",
-    "babel-core": "^6.26.3",
-    "bootstrap": "^4.3.1",
-    "concurrently": "^4.1.0",
-    "download": "^7.1.0",
-    "electron-dl": "^1.13.0",
-    "electron-reload": "^1.4.0",
-    "fs-extra": "^7.0.1",
-    "immutability-helper": "^3.0.0",
-    "jquery": "^3.4.0",
-    "lodash": "^4.17.11",
-    "node-sass": "^4.11.0",
-    "open-iconic": "^1.1.1",
-    "popper.js": "^1.15.0",
-    "rage-edit": "^1.2.0",
-    "react": "^16.8.6",
-    "react-dom": "^16.8.2",
-    "react-images": "^0.5.19",
-    "react-jsonschema-form": "^1.3.0",
-    "react-jsonschema-form-bs4": "^1.5.1",
-    "react-lightbox-component": "^1.2.1",
-    "react-markdown": "^4.0.6",
-    "react-redux": "^7.0.3",
-    "react-router-dom": "^4.3.1",
-    "redux": "^4.0.1",
-    "yarn": "^1.13.0",
-    "yauzl": "^2.10.0"
-  }
+	"name": "sc2-campaign-manager",
+	"version": "0.5.4-alpha",
+	"description": "A Campaign Manager for StarCraft II.  ",
+	"main": "main.js",
+	"build": {
+		"appId": "com.sc2mapster.campaignmanager",
+		"mac": {
+			"category": "demo.app"
+		},
+		"win": {
+			"icon": "build/s2icon.png",
+			"target": "nsis"
+		},
+		"nsis": {
+			"oneClick":false,
+			"perMachine":false, 
+			"allowToChangeInstallationDirectory":true
+
+		}
+	},
+	"scripts": {
+		"dev": "cross-env NODE_ENV=dev webpack --watch",
+		"start": "nf start",
+		"webpack": "webpack",
+		"electron": "cross-env NODE_ENV=dev electron .",
+		"build": "cross-env NODE_ENV=prod electron-builder"
+	},
+	"repository": "https://github.com/abrahamYG/sc2-campaign-manager",
+	"keywords": [
+		"Starcraft",
+		"demo"
+	],
+	"author": "abrahamYG",
+	"license": "CC0-1.0",
+	"devDependencies": {
+		"@babel/cli": "^7.2.3",
+		"@babel/core": "^7.4.3",
+		"@babel/plugin-proposal-class-properties": "^7.4.0",
+		"@babel/preset-env": "^7.3.1",
+		"@babel/preset-react": "^7.0.0",
+		"@babel/register": "^7.4.0",
+		"@fortawesome/fontawesome-free": "^5.8.1",
+		"awesome-typescript-loader": "^5.2.1",
+		"babel": "^6.23.0",
+		"babel-cli": "^6.26.0",
+		"babel-loader": "^8.0.5",
+		"babel-preset-flow": "^6.23.0",
+		"cross-env": "^5.2.0",
+		"css-loader": "^2.1.0",
+		"electron": "^4.0.4",
+		"electron-builder": "^20.41.0",
+		"electron-devtools-installer": "^2.2.4",
+		"font-awesome-loader": "^1.0.2",
+		"foreman": "^3.0.1",
+		"react-hot-loader": "^4.7.0",
+		"sass-loader": "^7.1.0",
+		"source-map-loader": "^0.2.4",
+		"style-loader": "^0.23.1",
+		"typescript": "^3.4.2",
+		"webpack": "^4.29.5",
+		"webpack-cli": "^3.2.3",
+		"webpack-dev-server": "^3.1.14"
+	},
+	"dependencies": {
+		"@types/debug": "^4.1.3",
+		"@types/download": "^6.2.4",
+		"@types/electron": "^1.6.10",
+		"@types/fs-extra": "^7.0.0",
+		"@types/lodash": "^4.14.130",
+		"@types/react": "^16.8.13",
+		"@types/react-dom": "^16.8.3",
+		"@types/react-images": "^0.5.1",
+		"@types/react-jsonschema-form": "^1.3.2",
+		"@types/react-redux": "^7.0.9",
+		"@types/react-router-dom": "^4.3.1",
+		"@types/yauzl": "^2.9.1",
+		"babel-core": "^6.26.3",
+		"bootstrap": "^4.3.1",
+		"concurrently": "^4.1.0",
+		"download": "^7.1.0",
+		"electron-dl": "^1.13.0",
+		"electron-reload": "^1.4.0",
+		"fs-extra": "^7.0.1",
+		"h2m": "^0.7.0",
+		"immutability-helper": "^3.0.0",
+		"jquery": "^3.4.0",
+		"lodash": "^4.17.11",
+		"node-sass": "^4.11.0",
+		"open-iconic": "^1.1.1",
+		"popper.js": "^1.15.0",
+		"rage-edit": "^1.2.0",
+		"react": "^16.8.6",
+		"react-dom": "^16.8.2",
+		"react-images": "^0.5.19",
+		"react-jsonschema-form": "^1.3.0",
+		"react-jsonschema-form-bs4": "^1.5.1",
+		"react-lightbox-component": "^1.2.1",
+		"react-markdown": "^4.0.6",
+		"react-redux": "^7.0.3",
+		"react-router-dom": "^4.3.1",
+		"redux": "^4.0.1",
+		"sc2mapster-crawler": "git+https://github.com/SC2Mapster/sc2mapster-crawler.git",
+		"yarn": "^1.13.0",
+		"yauzl": "^2.10.0"
+	}
 }

src/App.scss

@@ -91,4 +91,10 @@ progress[value]::-webkit-progress-value {
 
 .campaign-description img {
 	max-width: 100%;
+}
+
+.manifesteditor-views {
+	@extend .btn-group; 
+	@extend .mt-2;
+	@extend .float-right;
 }

src/classes/Campaign.ts

@@ -118,7 +118,9 @@ export default class Campaign {
 	}
 
 	static getCampaignsRemote = async (): Promise<Array<ICampaign>> => {
-		const campaigns:Array<ICampaign> = await Promise.all(Config.getSources().map((source:string) => Campaign.getCampaignRemote(source)));
+		const feed = Config.getFeed();
+		const sources:Array<string> = (feed !=="")?(await (await fetch(feed)).json()):Config.getSources()
+		const campaigns:Array<ICampaign> = await Promise.all(sources.map((source:string) => Campaign.getCampaignRemote(source)));
 		return campaigns;
 	}
 	static getCampaignsLocal = async ():Promise<Array<ICampaign>> => {

src/classes/Config.ts

@@ -7,11 +7,12 @@ const {currentPlatform, platforms} = require("./Platform")
 import path from 'path'
 import fs from 'fs'	
 
-const manifestSource = "https://raw.githubusercontent.com/abrahamYG/sc2-campaign-manager/master/public/sources.json";
+export const manifestFeed = "https://raw.githubusercontent.com/abrahamYG/sc2-campaign-manager/master/public/sources.json";
 export interface IConfig {
 	installDir:string, 
 	runCommand:string, 
 	runParams:string,
+	feed:string,
 	campaignSources:Array<string>
 	campaignLocalSources:Array<string>
 }
@@ -35,7 +36,7 @@ export default class Config {
 		return fs.existsSync(configFile);
 	}
 	static getSourcesRemote = async () => {
-		const response:Response = await fetch(manifestSource);
+		const response:Response = await fetch(manifestFeed);
 		const sources:Array<string> = await response.json();
 		return sources;
 	}
@@ -68,6 +69,9 @@ export default class Config {
 			//return this.installDir;
 		}
 	}
+	static getFeed():string {
+		return Config.configFileExists()&&Config.loadFromDisk().feed?(Config.loadFromDisk().feed):manifestFeed;
+	}
 	static getSources():Array<string> {
 		return  Config.configFileExists()&&Config.loadFromDisk().campaignSources? (Config.loadFromDisk().campaignSources):[""];
 	}

src/classes/Mapster.ts

@@ -0,0 +1,41 @@
+import {scrapper as mscrap} from  "sc2mapster-crawler";
+import Campaign, { ICampaign, IScreenshot, ISC2Component, ISC2Map } from "./Campaign";
+import path from 'path'
+import h2m from 'h2m'
+const conn = new mscrap.MapsterConnection();
+
+export const getCampaignFromMapster = async (projectId:string):Promise<ICampaign> => {
+    const project = await conn.getProjectOverview(projectId);
+    const screenshots:IScreenshot[] = (await conn.getProjectImages(projectId)).images.map(img =>({
+        src:img.imageUrl,
+        description:h2m(img.caption)
+    }));
+    let maps:ISC2Component[] = []
+
+    for await(const f of await conn.getProjectFilesList(projectId)){
+        const fileData = await conn.getProjectFile(projectId,f.id);
+        const map:ISC2Component = {
+            name:fileData.title,
+            description:h2m(fileData.description.html),
+            destination:`maps/${fileData.filename}`,
+            source:`https://www.sc2mapster.com/projects/${projectId}/files/${fileData.id}/download`,
+            sourceFormat:path.extname(fileData.filename),
+            fileEntry:""
+        }
+        maps.push(map);
+    }
+    return {
+        ...Campaign.emptyCampaign(),
+        id: projectId,
+        name:project.base.name,
+        description: h2m(project.description.html),
+        author:project.owner.name,
+        thumbnail:project.base.thumbnail,
+        lastUpdated:project.updatedAt.toISOString(),
+        patchNotes:[],
+        videos:[],
+        screenshots,
+        maps,
+        mods:[]
+    }
+} 

src/components/ManifestEditor.tsx

@@ -9,7 +9,7 @@ import { HashRouter as Router, Route , NavLink, Redirect } from 'react-router-do
 import CampaignDetails from './CampaignDetails';
 import ManifestEditorScreenshot from './ManifestEditorScreenshotItem';
 import ManifestEditorJson from './ManifestEditorJson';
-
+import _ from 'lodash'
 
 interface IManifestEditorProps{
 	"campaign"?:ICampaign,
@@ -69,6 +69,13 @@ const ManifestEditor:FC<IManifestEditorProps> = (props) => {
 		};
 		setCampaign(newCampaign,index);
 	}
+	const removeMap = (maps:ISC2Map[],mapIndex:number) =>{
+		setCampaign({
+			...campaign,
+			maps:_.xor(maps, [maps[mapIndex]])
+		},index);
+	};
+
 
 	const setCampaignMod = (mod:ISC2Mod,modIndex:number)=>{
 		const mods = [...campaign.mods];
@@ -98,14 +105,14 @@ const ManifestEditor:FC<IManifestEditorProps> = (props) => {
 			<Route exact path="/" render={() => (
 					<Redirect to="/editor"/>
 			)}/>
-			<div className="btn-group mt-2 float-right" role="group" aria-label="">
-				<NavLink to="/editor" className="btn btn-outline-primary" activeClassName="btn-primary active">
+			<div className="manifesteditor-views" role="group" aria-label="">
+				<NavLink to="/editor" className="manifesteditor-viewoption btn btn-outline-primary" activeClassName="btn-primary active">
 					Editor
 				</NavLink>
-				<NavLink to="/json" className="btn btn-outline-primary" activeClassName="btn-primary active">
+				<NavLink to="/json" className="manifesteditor-viewoption btn btn-outline-primary" activeClassName="btn-primary active">
 					JSON
 				</NavLink>
-				<NavLink to="/preview" className="btn btn-outline-primary" activeClassName="btn-primary active">
+				<NavLink to="/preview" className="manifesteditor-viewoption btn btn-outline-primary" activeClassName="btn-primary active">
 					Preview
 				</NavLink>
 			</div>

src/components/NavBar.tsx

@@ -31,7 +31,12 @@ function NavBar(props:NavBarProps) {
 					<NavLink to="/mapmakers" className="nav-link" activeClassName="active">
 						Manifest Editor
 					</NavLink>
-				</li>    
+				</li>
+				<li className="nav-item">
+					<NavLink to="/scrapper" className="nav-link" activeClassName="active">
+						Mapster Scrapper
+					</NavLink>
+				</li> 
 				</ul>
 				{/*
 				<ul className="navbar-nav">

src/components/SettingsForm.tsx

@@ -9,6 +9,7 @@ const SettingsForm = (props:any) => {
 		installDir, setInstallDir,
 		runCommand, setRunCommand,
 		runParams, setRunParams,
+		feed, setFeed,
 		campaignSources, setCampaignSources,
 		campaignLocalSources, setCampaignLocalSources,
 		disabledForm, setDisabledForm,
@@ -90,6 +91,20 @@ const SettingsForm = (props:any) => {
 					These are parameters used to run the map and set additional settings when running StarcCraft II.
 					If you are not sure what to write here, you can happily leave the default values.
 				</small>
+
+				<label htmlFor="feed">Campaign Feed</label>
+				<input 
+					value={feed}
+					onChange={e => setFeed(e.target.value)}
+					type="text" 
+					className="form-control" 
+					name="feed" 
+					aria-describedby="feedHelp" 
+					placeholder="Enter a set of parameters" 
+				/>
+				<small id="feedHelp" className="form-text text-muted">
+					A remote feed that provides a set of campaigns available to download
+				</small>
 				<label htmlFor="campaignSources">Campaign Sources</label>
 				<textarea 
 					name="campaignSources"