Added cryptocompare impl + file renamer

ccrename.js

@@ -0,0 +1,20 @@
+import fs from 'fs-extra'
+
+let folder = './data/CCCAGG/BTC/USD/'
+
+const read = file=> fs.readFile( folder+file, 'utf8' )
+//.then( getTimeline )
+const parseCSV = data=> data.split('\n').map( s=> s.split(',') )
+const getTime = values=> values[0]
+
+fs.readdir( folder )
+	.then( files=> files.map( file=> {
+		let tl = fs.readFile( folder+file, 'utf8' )
+				.then( parseCSV )
+				.then( csv=> csv[csv.length-1][0] )
+				.then( last=>
+		fs.move( folder+file, folder+file.replace(/\.csv$/, '-'+last+'.csv') )
+				)
+		})
+	)
+

providers/cryptocompare.js

@@ -1,7 +1,12 @@
-import fetch from 'node-fetch'
+import config from '../config'
+import fetch from 'fetch-filecache-for-crawling'
+//fetch.setParameter( 'logToConsole', true )
+fetch.setParameter( 'cacheFolder', `${config.dataDir}/fetched/` )
+global.fetch = fetch
+
 import cc from 'cryptocompare'
 import fs from 'fs-extra'
-global.fetch = fetch
+import { log } from '../utils/log'
 
 cc.rateLimit = function( apiKey )
 {
@@ -24,8 +29,58 @@ export function loadByPath( ccfs )
 const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 }
 const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 }
 
+// esm providers/cryptocompare:coins BTC ETH
+// coins([ 'BTC', 'ETH' ]).then
+export async function coins( syms )
+{
+	let list = await cc.coinList()
+	log('args')(syms)
+	return Promise.all(
+	Object.keys( list.Data )
+                .filter( sym=> syms.length? ~syms.indexOf(sym): true )
+		.map( sym=> {
+			fs.outputFile(`${config.dataDir}/coins/${list.Data[sym].Symbol}/index.json`, JSON.stringify(list.Data[sym]) )
+			return fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'})
+				.then( res=> res.text() )
+				.then(
+        	                        html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
+	                        )
+				.then( json=> (fs.outputFile(`${config.dataDir}/coins/${sym}/overview.json`, json), json) )
+				.then(
+	                                str=> JSON.parse( str )
+	                        )
+
+		})
+/*
+//		.map( log() )
+		.map( coin=> (fs.outputFile(`${config.dataDir}/coins/${list.Data[coin].Symbol}/index.json`, JSON.stringify(list.Data[coin])),coin) )
+		.filter( sym=> syms.length? ~syms.indexOf(sym): true )
+		.map( log('selected') )
+		.map( sym=> fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'}) )
+		.map( prom=> prom.then( res=> res.text() ) )
+		.map( prom=> prom.then(
+				html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
+			)
+		)
+                .map( prom=> prom.then( coin=>(fs.outputFile(`${config.dataDir}/coins/${coin.Data.General.Symbol}/overview.json`,coin),coin) ) )
+		.map( prom=> prom.then(
+				html=> JSON.parse( html )
+			)
+//			.then(log('ress'))
+//			.then(coin=>fs.outputFile(`${config.dataDir}/coins/${coin.Symbol}/infos.json`,coin))
+		)
+/*
+		.map( html=> ({
+			name: html.match(//)[1]
+		,	startDate: html.match(//)[1]
+		}))
+*/
+	).then( ress=> ress.reduce((all,json)=>({...all,[json.Data.General.Symbol]:json}), {}) )
+}
+
+
 
 export {
-	coinList as coins
+	coinList as coinsL
 ,	exchangeList as exchanges
 } from 'cryptocompare'

utils/fetchWithCache.js

@@ -0,0 +1,7 @@
+import fetch from 'node-fetch'
+import config from '../config'
+
+global.fetch = (...args)=> fetch( ...args )
+				.then( res=> (res.text().then( txt=> fs.outputFile(`${config.dataDir}/fetched/${Date.now()}`)))
+
+

utils/log.js

@@ -0,0 +1 @@
+export const log = (...args)=> obj=> ( console.log( ...args, obj ), obj )