#Python script
#This will load all data as a Pandas dataframe
#courtesy of Abyrint.com
#MIT license
import pandas as pd
ep = endpoint //get endpoint from the tab
#reads the index api
tableType = 'dbSomaliaFgsRevExp' #customize this to the type of table required 
#reads the indexfile
idx = pd.read_csv(ep,sep='\t', header=0)
url_list =[]
for index, rows in idx.iterrows():
    if rows.standardName == tableType: #change for alternative conditions
        my_list =rows.tsv
    url_list.append(my_list)
#loads all
dfs = [pd.read_csv(url,sep='\t', header=0) for url in url_list]
df = pd.concat(dfs, ignore_index=True)
#Do analysis with the compiled datatable here
print(df.shape)
             
                                            

//Javascript, Node.js
//This will compile data for all years available as an array of objects in JSON format.
//Each object in the array holds key/value pairs that matches the headers. 
//courtesy of Abyrint.com
// MIT license
const url = endpoint //get url from endpoint tab
const fetch = require('node-fetch');
const tableType = 'dbSomaliaFgsRevExp' //customize this to the type of table required
//const year = '2021' //use this for alternative filter
//builds compiled JSON object for all tables of this type
bigArr = []
const dataset = getData(url).then(x => {
    var index = tToJson(x)
    index.forEach(function (row) {
        if (row.standardName == tableType) { // && row.year >= year //add to filter if required
            getData(row.tsv).then(y => {
                var data = tToJson(y)
                data.forEach(function (q) { bigArr.push(q) })
                console.log(bigArr.length)
                // Do anything with the dataset here
                // bigArr
            })
        }
    })
})

function getData(url) {
    try {
        return fetch(url).then(res => { return res.text() }).then(body => {return body})
    } catch (err) {
        alert(err);
    }
}
//TSV parser to JSON
function tToJson(str) {
    const lines = str.replace(/\r\n?/g, "\n").split("\n");
    const headers = lines.shift().split('\t');
    return lines.map(line => {
        const data = line.split('\t');
        return headers.reduce((o, k, i) => {
            o[k] = data[i];
            return o;
        }, {});
    });
}                 
                                                                    

const endpoint = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vSAu1M-o9hgbZIKlvliDeqXle1LHWN82Li0xPrpzkudj1ACKLsgoIKESgQ5Si7fSAwwrlFIgU0kHkun/pub?output=tsv'