Added cryptocompare impl + file renamer
This commit is contained in:
parent
50b70cf6fb
commit
a7d41ca69b
|
@ -0,0 +1,20 @@
|
|||
import fs from 'fs-extra'
|
||||
|
||||
let folder = './data/CCCAGG/BTC/USD/'
|
||||
|
||||
const read = file=> fs.readFile( folder+file, 'utf8' )
|
||||
//.then( getTimeline )
|
||||
const parseCSV = data=> data.split('\n').map( s=> s.split(',') )
|
||||
const getTime = values=> values[0]
|
||||
|
||||
fs.readdir( folder )
|
||||
.then( files=> files.map( file=> {
|
||||
let tl = fs.readFile( folder+file, 'utf8' )
|
||||
.then( parseCSV )
|
||||
.then( csv=> csv[csv.length-1][0] )
|
||||
.then( last=>
|
||||
fs.move( folder+file, folder+file.replace(/\.csv$/, '-'+last+'.csv') )
|
||||
)
|
||||
})
|
||||
)
|
||||
|
|
@ -1,7 +1,12 @@
|
|||
import fetch from 'node-fetch'
|
||||
import config from '../config'
|
||||
import fetch from 'fetch-filecache-for-crawling'
|
||||
//fetch.setParameter( 'logToConsole', true )
|
||||
fetch.setParameter( 'cacheFolder', `${config.dataDir}/fetched/` )
|
||||
global.fetch = fetch
|
||||
|
||||
import cc from 'cryptocompare'
|
||||
import fs from 'fs-extra'
|
||||
global.fetch = fetch
|
||||
import { log } from '../utils/log'
|
||||
|
||||
cc.rateLimit = function( apiKey )
|
||||
{
|
||||
|
@ -24,8 +29,58 @@ export function loadByPath( ccfs )
|
|||
const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 }
|
||||
const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 }
|
||||
|
||||
// esm providers/cryptocompare:coins BTC ETH
|
||||
// coins([ 'BTC', 'ETH' ]).then
|
||||
export async function coins( syms )
|
||||
{
|
||||
let list = await cc.coinList()
|
||||
log('args')(syms)
|
||||
return Promise.all(
|
||||
Object.keys( list.Data )
|
||||
.filter( sym=> syms.length? ~syms.indexOf(sym): true )
|
||||
.map( sym=> {
|
||||
fs.outputFile(`${config.dataDir}/coins/${list.Data[sym].Symbol}/index.json`, JSON.stringify(list.Data[sym]) )
|
||||
return fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'})
|
||||
.then( res=> res.text() )
|
||||
.then(
|
||||
html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
|
||||
)
|
||||
.then( json=> (fs.outputFile(`${config.dataDir}/coins/${sym}/overview.json`, json), json) )
|
||||
.then(
|
||||
str=> JSON.parse( str )
|
||||
)
|
||||
|
||||
})
|
||||
/*
|
||||
// .map( log() )
|
||||
.map( coin=> (fs.outputFile(`${config.dataDir}/coins/${list.Data[coin].Symbol}/index.json`, JSON.stringify(list.Data[coin])),coin) )
|
||||
.filter( sym=> syms.length? ~syms.indexOf(sym): true )
|
||||
.map( log('selected') )
|
||||
.map( sym=> fetch(`https://www.cryptocompare.com/${list.Data[sym].Url}`, {refresh:'never'}) )
|
||||
.map( prom=> prom.then( res=> res.text() ) )
|
||||
.map( prom=> prom.then(
|
||||
html=> html.match(/pageInfo\.setCoinPageInfo\((.*?)\);/m)[1]
|
||||
)
|
||||
)
|
||||
.map( prom=> prom.then( coin=>(fs.outputFile(`${config.dataDir}/coins/${coin.Data.General.Symbol}/overview.json`,coin),coin) ) )
|
||||
.map( prom=> prom.then(
|
||||
html=> JSON.parse( html )
|
||||
)
|
||||
// .then(log('ress'))
|
||||
// .then(coin=>fs.outputFile(`${config.dataDir}/coins/${coin.Symbol}/infos.json`,coin))
|
||||
)
|
||||
/*
|
||||
.map( html=> ({
|
||||
name: html.match(//)[1]
|
||||
, startDate: html.match(//)[1]
|
||||
}))
|
||||
*/
|
||||
).then( ress=> ress.reduce((all,json)=>({...all,[json.Data.General.Symbol]:json}), {}) )
|
||||
}
|
||||
|
||||
|
||||
|
||||
export {
|
||||
coinList as coins
|
||||
coinList as coinsL
|
||||
, exchangeList as exchanges
|
||||
} from 'cryptocompare'
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
import fetch from 'node-fetch'
|
||||
import config from '../config'
|
||||
|
||||
global.fetch = (...args)=> fetch( ...args )
|
||||
.then( res=> (res.text().then( txt=> fs.outputFile(`${config.dataDir}/fetched/${Date.now()}`)))
|
||||
|
||||
|
|
@ -0,0 +1 @@
|
|||
export const log = (...args)=> obj=> ( console.log( ...args, obj ), obj )
|
Loading…
Reference in New Issue