diff --git a/ccrename.js b/ccrename.js new file mode 100644 index 0000000..82bbad7 --- /dev/null +++ b/ccrename.js @@ -0,0 +1,20 @@ +import fs from 'fs-extra' + +let folder = './data/CCCAGG/BTC/USD/' + +const read = file=> fs.readFile( folder+file, 'utf8' ) +//.then( getTimeline ) +const parseCSV = data=> data.split('\n').map( s=> s.split(',') ) +const getTime = values=> values[0] + +fs.readdir( folder ) + .then( files=> files.map( file=> { + let tl = fs.readFile( folder+file, 'utf8' ) + .then( parseCSV ) + .then( csv=> csv[csv.length-1][0] ) + .then( last=> + fs.move( folder+file, folder+file.replace(/\.csv$/, '-'+last+'.csv') ) + ) + }) + ) + diff --git a/providers/cryptocompare.js b/providers/cryptocompare.js index 1161ac0..947f7b3 100644 --- a/providers/cryptocompare.js +++ b/providers/cryptocompare.js @@ -1,7 +1,12 @@ -import fetch from 'node-fetch' +import config from '../config' +import fetch from 'fetch-filecache-for-crawling' +//fetch.setParameter( 'logToConsole', true ) +fetch.setParameter( 'cacheFolder', `${config.dataDir}/fetched/` ) +global.fetch = fetch + import cc from 'cryptocompare' import fs from 'fs-extra' -global.fetch = fetch +import { log } from '../utils/log' cc.rateLimit = function( apiKey ) { @@ -24,8 +29,58 @@ export function loadByPath( ccfs ) const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 } const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 } +// esm providers/cryptocompare:coins BTC ETH +// coins([ 'BTC', 'ETH' ]).then +export async function coins( syms ) +{ + let list = await cc.coinList() + log('args')(syms) + return Promise.all( + Object.keys( list.Data ) + .filter( sym=> syms.length? ~syms.indexOf(sym): true ) + .map( sym=> { + fs.outputFile(`${config.dataDir}/coins/${list.Data[sym].Symbol}/index.json`, JSON.stringify(list.Data[sym]) ) + return fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'}) + .then( res=> res.text() ) + .then( + html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1] + ) + .then( json=> (fs.outputFile(`${config.dataDir}/coins/${sym}/overview.json`, json), json) ) + .then( + str=> JSON.parse( str ) + ) + + }) +/* +// .map( log() ) + .map( coin=> (fs.outputFile(`${config.dataDir}/coins/${list.Data[coin].Symbol}/index.json`, JSON.stringify(list.Data[coin])),coin) ) + .filter( sym=> syms.length? ~syms.indexOf(sym): true ) + .map( log('selected') ) + .map( sym=> fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'}) ) + .map( prom=> prom.then( res=> res.text() ) ) + .map( prom=> prom.then( + html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1] + ) + ) + .map( prom=> prom.then( coin=>(fs.outputFile(`${config.dataDir}/coins/${coin.Data.General.Symbol}/overview.json`,coin),coin) ) ) + .map( prom=> prom.then( + html=> JSON.parse( html ) + ) +// .then(log('ress')) +// .then(coin=>fs.outputFile(`${config.dataDir}/coins/${coin.Symbol}/infos.json`,coin)) + ) +/* + .map( html=> ({ + name: html.match(//)[1] + , startDate: html.match(//)[1] + })) +*/ + ).then( ress=> ress.reduce((all,json)=>({...all,[json.Data.General.Symbol]:json}), {}) ) +} + + export { - coinList as coins + coinList as coinsL , exchangeList as exchanges } from 'cryptocompare' diff --git a/utils/fetchWithCache.js b/utils/fetchWithCache.js new file mode 100644 index 0000000..d4fd8a3 --- /dev/null +++ b/utils/fetchWithCache.js @@ -0,0 +1,7 @@ +import fetch from 'node-fetch' +import config from '../config' + +global.fetch = (...args)=> fetch( ...args ) + .then( res=> (res.text().then( txt=> fs.outputFile(`${config.dataDir}/fetched/${Date.now()}`))) + + diff --git a/utils/log.js b/utils/log.js new file mode 100644 index 0000000..f4bcb7c --- /dev/null +++ b/utils/log.js @@ -0,0 +1 @@ +export const log = (...args)=> obj=> ( console.log( ...args, obj ), obj )