that-hypermnesiac-guy/providers/cryptocompare.js

import config from '../config'
import fetch from 'fetch-filecache-for-crawling'
//fetch.setParameter( 'logToConsole', true )
fetch.setParameter( 'cacheFolder', `${config.dataDir}/fetched/` )
global.fetch = fetch

import cc from 'cryptocompare'
import fs from 'fs-extra'
import { log } from '../utils/log'

cc.rateLimit = function( apiKey )
{
	return fetch( `https://min-api.cryptocompare.com/stats/rate/limit${apiKey ? `?api_key=${apiKey}` : ''}` )
		.then( res=> { if( !res.ok ){ throw new Error(`${res.status} ${res.statusText}`) }else return res.json() })
		.then( body=> { if( body.Response === 'Error' ){ throw body.Message }else return body })
}

// loadByPath( "EXCHANGE/SYM/SYM/PERIOD/TS|/TS|LIMIT" ).then()
export function loadByPath( ccfs )
{
	let [ exchange, fsym, tsym, period, start, limit = 2000 ] = ccfs.split("/")
	,       [ , aggregate, tUnit ] = period.match( /(\d+)([HDWM]*)/ )
	,       loader = 'histo' + (tUnit == '' ? 'Minute' : tUnit == 'H' ? 'Hour' : 'Day')
	// calculate toTs from limit and start
	console.log( loader,  aggregate*timeFrames[tUnit] )
	return cc[loader]( fsym, tsym, { allData: true, limit: 2000, aggregate: aggregate*timeFrames[tUnit], exchange } )
}

const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 }
const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 }

// esm providers/cryptocompare:coins BTC ETH
// coins([ 'BTC', 'ETH' ]).then
export async function coins( syms )
{
	let list = await cc.coinList()
	log('args')(syms)
	return Promise.all(
	Object.keys( list.Data )
                .filter( sym=> syms.length? ~syms.indexOf(sym): true )
		.map( sym=> {
			fs.outputFile(`${config.dataDir}/coins/${list.Data[sym].Symbol}/index.json`, JSON.stringify(list.Data[sym]) )
			return fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'})
				.then( res=> res.text() )
				.then(
        	                        html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
	                        )
				.then( json=> (fs.outputFile(`${config.dataDir}/coins/${sym}/overview.json`, json), json) )
				.then(
	                                str=> JSON.parse( str )
	                        )

		})
/*
//		.map( log() )
		.map( coin=> (fs.outputFile(`${config.dataDir}/coins/${list.Data[coin].Symbol}/index.json`, JSON.stringify(list.Data[coin])),coin) )
		.filter( sym=> syms.length? ~syms.indexOf(sym): true )
		.map( log('selected') )
		.map( sym=> fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'}) )
		.map( prom=> prom.then( res=> res.text() ) )
		.map( prom=> prom.then(
				html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
			)
		)
                .map( prom=> prom.then( coin=>(fs.outputFile(`${config.dataDir}/coins/${coin.Data.General.Symbol}/overview.json`,coin),coin) ) )
		.map( prom=> prom.then(
				html=> JSON.parse( html )
			)
//			.then(log('ress'))
//			.then(coin=>fs.outputFile(`${config.dataDir}/coins/${coin.Symbol}/infos.json`,coin))
		)
/*
		.map( html=> ({
			name: html.match(//)[1]
		,	startDate: html.match(//)[1]
		}))
*/
	).then( ress=> ress.reduce((all,json)=>({...all,[json.Data.General.Symbol]:json}), {}) )
}



export {
	coinList as coinsL
,	exchangeList as exchanges
} from 'cryptocompare'