all_containers/common_files/pheno_meta-data_excel_to_rdf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup and helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import rdflib\n",
    "from rdflib import URIRef\n",
    "import os\n",
    "import csv\n",
    "import pprint\n",
    "import hashlib\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "\n",
    "data_or = 'data-original' + os.sep + 'phenotypic' + os.sep\n",
    "data_gen = 'data-generated' + os.sep + 'phenotypic' + os.sep\n",
    "\n",
    "if not os.path.exists('data-generated'):\n",
    "    os.mkdir('data-generated')\n",
    "if not os.path.exists(data_gen):\n",
    "    os.mkdir(data_gen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "code_folding": [
     0,
     7,
     9
    ]
   },
   "outputs": [],
   "source": [
    "def reshape_sheet(sheet):\n",
    "    new_sheet = sheet.where(pd.notnull(sheet), None)\n",
    "    new_sheet = new_sheet.drop([0,1,2])\n",
    "    del(new_sheet['Field'])\n",
    "    new_sheet.reset_index(inplace=True, drop=True)\n",
    "    return(new_sheet)\n",
    "\n",
    "def make_safe(s):\n",
    "    safe_s = ''\n",
    "    for c in s:\n",
    "        safe_s += c if c.isalnum() else '_'\n",
    "    return safe_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def parseStudyData(filename, df, nodes):\n",
    "    g = rdflib.Graph()  # Create RDF graph  \n",
    "    # Define prefixes\n",
    "    rdf = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')\n",
    "    rdfs = rdflib.Namespace('http://www.w3.org/2000/01/rdf-schema#')\n",
    "    ppeo = rdflib.Namespace('http://purl.org/ppeo/PPEO.owl#')\n",
    "    g.bind('rdf', rdf)\n",
    "    g.bind('rdfs', rdfs)\n",
    "    g.bind('ppeo', ppeo)\n",
    "    \n",
    "    _, study_ID = filename.split('_')\n",
    "    study_ID, _ = study_ID.split('.')\n",
    "    fileNode = nodes['datafiles'][filename]\n",
    "    varColumns = list(df.columns)\n",
    "    varColumns.remove('Observation unit ID')\n",
    "    varColumns.remove('Date')\n",
    "    counter_obs_ID = 0\n",
    "    for index, row in df.iterrows():\n",
    "        curDate = rdflib.Literal(row['Date'], datatype=rdflib.XSD.date)\n",
    "        curOu = nodes['ou'][row['Observation unit ID']]\n",
    "        for var in varColumns: \n",
    "            # ignore if there is no value\n",
    "            if row[var]:\n",
    "                if var in nodes['ov']:\n",
    "                    varNode = nodes['ov'][var.strip()]\n",
    "                else:\n",
    "                    print('Variable not found: ' + var)\n",
    "                    continue\n",
    "                #curVar = 1\n",
    "                counter_obs_ID += 1\n",
    "                obsID_base_string = ('#observation/' + nodes['investigation_ID'] + '_' + study_ID + \n",
    "                                     '_' + make_safe(var))\n",
    "                obsNode = URIRef(obsID_base_string + '_' + str(counter_obs_ID))  # rdflib.BNode() \n",
    "                g.add((\n",
    "                    obsNode,\n",
    "                    rdf.type,\n",
    "                    ppeo.observation\n",
    "                ))\n",
    "                g.add((\n",
    "                    fileNode,\n",
    "                    ppeo.hasObservation,\n",
    "                    obsNode\n",
    "                ))\n",
    "                g.add((\n",
    "                    fileNode,\n",
    "                    ppeo.hasDigitalLocation,\n",
    "                    rdflib.Literal(filename)\n",
    "                ))\n",
    "                g.add((\n",
    "                    obsNode,\n",
    "                    ppeo.hasObservedSubject,\n",
    "                    curOu\n",
    "                ))\n",
    "                g.add((\n",
    "                    obsNode,\n",
    "                    ppeo.hasVariable,\n",
    "                    varNode\n",
    "                ))\n",
    "                g.add((\n",
    "                    obsNode,\n",
    "                    ppeo.hasDateTime,\n",
    "                    curDate\n",
    "                ))\n",
    "                g.add((\n",
    "                    obsNode,\n",
    "                    ppeo.hasValue,\n",
    "                    rdflib.Literal(row[var])\n",
    "                ))\n",
    "    return g"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MIAPPE metadata to ttl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Read sheets from file\n",
    "inv_sheet = pd.read_excel('data-original' + os.sep + 'MIAPPE_CxE_v1.1.xlsx', sheet_name='Investigation')\n",
    "study_sheet = pd.read_excel('data-original' + os.sep + 'MIAPPE_CxE_v1.1.xlsx', sheet_name='Study')\n",
    "person_sheet = pd.read_excel('data-original' + os.sep + 'MIAPPE_CxE_v1.1.xlsx', sheet_name='Person')\n",
    "datafile_sheet = pd.read_excel('data-original' + os.sep + 'MIAPPE_CxE_v1.1.xlsx', sheet_name='Data file')\n",
    "bm_sheet = pd.read_excel('data-original' + os.sep + 'MIAPPE_CxE_v1.1.xlsx', sheet_name='Biological Material')\n",
    "ou_sheet = pd.read_excel('data-original' + os.sep + 'MIAPPE_CxE_v1.1.xlsx', sheet_name='Observation Unit')\n",
    "ov_sheet = pd.read_excel('data-original' + os.sep + 'MIAPPE_CxE_v1.1.xlsx', sheet_name='Observed Variable')\n",
    "\n",
    "# drop unnecessary columns/rows\n",
    "study_sheet = reshape_sheet(study_sheet)\n",
    "person_sheet = reshape_sheet(person_sheet)\n",
    "datafile_sheet = reshape_sheet(datafile_sheet)\n",
    "bm_sheet = reshape_sheet(bm_sheet)\n",
    "ou_sheet = reshape_sheet(ou_sheet)\n",
    "ov_sheet = reshape_sheet(ov_sheet)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": [
     0,
     25,
     31,
     37,
     43,
     46,
     52,
     55,
     61,
     64,
     70,
     73,
     79,
     82,
     88,
     91,
     97
    ]
   },
   "outputs": [],
   "source": [
    "# Create graph, parse investigation\n",
    "\n",
    "nodes = {}\n",
    "\n",
    "# Define prefixes\n",
    "rdf = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')\n",
    "rdfs = rdflib.Namespace('http://www.w3.org/2000/01/rdf-schema#')\n",
    "ppeo = rdflib.Namespace('http://purl.org/ppeo/PPEO.owl#')\n",
    "base = rdflib.Namespace('http://localhost/wur/fair-example#')\n",
    "\n",
    "g = rdflib.Graph()  # Create RDF graph\n",
    "\n",
    "g.bind('rdf', rdf)\n",
    "g.bind('rdfs', rdfs)\n",
    "g.bind('ppeo', ppeo)\n",
    "\n",
    "# Investigation\n",
    "inv_ID = 'phInvID'\n",
    "if inv_sheet[['Value'][0]][0]:\n",
    "    inv_ID = make_safe(inv_sheet[['Value'][0]][0].strip())\n",
    "else:\n",
    "    print('INFO: no investigation ID')\n",
    "\n",
    "invNode = URIRef('#investigation/' + make_safe(inv_ID))  # rdflib.BNode()\n",
    "nodes = {'investigation': invNode, 'investigation_ID': make_safe(inv_ID)}\n",
    "g.add((\n",
    "    invNode,\n",
    "    rdf.type,\n",
    "    ppeo.investigation\n",
    "))    \n",
    "    \n",
    "g.add((\n",
    "    invNode,\n",
    "    ppeo.hasIdentifier,\n",
    "    rdflib.Literal(inv_ID)\n",
    "))\n",
    "    \n",
    "if inv_sheet[['Value'][0]][1]:    \n",
    "    g.add((\n",
    "        invNode,\n",
    "        ppeo.hasName,\n",
    "        rdflib.Literal(inv_sheet[['Value'][0]][1])\n",
    "    ))\n",
    "else:\n",
    "    print('ERROR: no investigation title')\n",
    "\n",
    "if inv_sheet[['Value'][0]][2]:  \n",
    "    g.add((\n",
    "        invNode,\n",
    "        ppeo.hasDescription,\n",
    "        rdflib.Literal(inv_sheet[['Value'][0]][2])\n",
    "    ))\n",
    "else:\n",
    "    print('INFO: no investigation description')\n",
    "    \n",
    "if inv_sheet[['Value'][0]][3]:  \n",
    "    g.add((\n",
    "        invNode,\n",
    "        ppeo.hasSubmissionDate,\n",
    "        rdflib.Literal(inv_sheet[['Value'][0]][3], datatype=rdflib.XSD.date)\n",
    "    ))\n",
    "else:\n",
    "    print('INFO: no investigation submission date')   \n",
    "    \n",
    "if inv_sheet[['Value'][0]][4]:  \n",
    "    g.add((\n",
    "        invNode,\n",
    "        ppeo.hasPublicReleaseDate,\n",
    "        rdflib.Literal(inv_sheet[['Value'][0]][4], datatype=rdflib.XSD.date)\n",
    "    ))\n",
    "else:\n",
    "    print('INFO: no investigation release date')\n",
    "    \n",
    "if inv_sheet[['Value'][0]][5]:  \n",
    "    g.add((\n",
    "        invNode,\n",
    "        ppeo.hasLicense,\n",
    "        rdflib.Literal(inv_sheet[['Value'][0]][5])\n",
    "    ))\n",
    "else:\n",
    "    print('INFO: no investigation license')\n",
    "\n",
    "if inv_sheet[['Value'][0]][6]:  \n",
    "    g.add((\n",
    "        invNode,\n",
    "        ppeo.hasMIAPPEVersion,\n",
    "        rdflib.Literal(inv_sheet[['Value'][0]][6], datatype=rdflib.XSD.float)\n",
    "    ))\n",
    "else:\n",
    "    print('INFO: no investigation miappe version')\n",
    "    \n",
    "if inv_sheet[['Value'][0]][7]:  \n",
    "    g.add((\n",
    "        invNode,\n",
    "        ppeo.hasAssociatedPublication,\n",
    "        rdflib.Literal(inv_sheet[['Value'][0]][7])\n",
    "    ))\n",
    "else:\n",
    "    print('INFO: no investigation publications')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "code_folding": [
     0,
     13,
     86,
     98,
     103,
     108,
     113,
     135,
     140,
     145,
     150,
     157,
     164,
     169,
     174,
     180,
     186,
     196,
     201,
     250,
     255,
     260,
     265
    ]
   },
   "outputs": [],
   "source": [
    "# Study\n",
    "nodes['studies'] = {}\n",
    "nodes['obsLevelHierarchy'] = {}\n",
    "nodes['spatDist'] = {}\n",
    "for study in study_sheet['Study unique ID*']:\n",
    "    stuIDstp = study.strip()\n",
    "    nodes['studies'][stuIDstp] = URIRef('#study/' + inv_ID + '_' + make_safe(stuIDstp))  # rdflib.BNode()\n",
    "    nodes['obsLevelHierarchy'][stuIDstp] = {}\n",
    "    nodes['spatDist'][stuIDstp] = {'study': '#spatial_distribution/' + inv_ID + '_' + \n",
    "                                   make_safe(stuIDstp) + '_spatial_distribution'}\n",
    "    \n",
    "nodes['countries'] = {}  \n",
    "nodes['locations'] = {}\n",
    "locationColumns = zip(\n",
    "    study_sheet['Geographic location (country)*'], \n",
    "    study_sheet['Experimental site name*'],\n",
    "    study_sheet['Geographic location (latitude)'], \n",
    "    study_sheet['Geographic location (longitude)'], \n",
    "    study_sheet['Geographic location (altitude)']\n",
    ")\n",
    "for countryStr, locStr, latStr, lonStr, altStr in locationColumns:\n",
    "    country = countryStr.strip()\n",
    "    if country not in nodes['countries']:\n",
    "        nodes['countries'][country] = URIRef('#country/' + make_safe(country))  # rdflib.BNode() \n",
    "        g.add((\n",
    "            nodes['countries'][country],\n",
    "            ppeo.hasName,\n",
    "            rdflib.Literal(country)\n",
    "        ))\n",
    "        g.add((\n",
    "            nodes['countries'][country],\n",
    "            rdf.type,\n",
    "            ppeo.country\n",
    "        ))\n",
    "    \n",
    "    loc = locStr.strip()\n",
    "    if loc not in nodes['locations']:\n",
    "        loc_hash = int(hashlib.sha1(loc.encode('utf-8')).hexdigest(), 16) % (10 ** 8)\n",
    "        nodes['locations'][loc] = URIRef('#location/' + str(loc_hash))  # rdflib.BNode() \n",
    "        g.add((\n",
    "            nodes['locations'][loc],\n",
    "            rdf.type,\n",
    "            ppeo.location\n",
    "        ))\n",
    "        g.add((\n",
    "            nodes['locations'][loc],\n",
    "            ppeo.hasCountry,\n",
    "            nodes['countries'][country]\n",
    "        ))\n",
    "        locParts = [x.strip() for x in loc.split(';')]\n",
    "\n",
    "        g.add((\n",
    "            nodes['locations'][loc],\n",
    "            ppeo.hasName,\n",
    "            rdflib.Literal(locParts[1])\n",
    "        ))\n",
    "        g.add((\n",
    "            nodes['locations'][loc],\n",
    "            ppeo.hasAddress,\n",
    "            rdflib.Literal(locParts[0])\n",
    "        ))\n",
    "\n",
    "        lat, long, alt = float(latStr), float(lonStr), altStr.strip()\n",
    "        g.add((\n",
    "            nodes['locations'][loc],\n",
    "            ppeo.hasLatitude,\n",
    "            rdflib.Literal(lat, datatype=rdflib.XSD.float)\n",
    "        ))\n",
    "        g.add((\n",
    "            nodes['locations'][loc],\n",
    "            ppeo.hasLongitude,\n",
    "            rdflib.Literal(long, datatype=rdflib.XSD.float)\n",
    "        ))            \n",
    "        g.add((\n",
    "            nodes['locations'][loc],\n",
    "            ppeo.hasAltitude,\n",
    "            rdflib.Literal(alt, datatype=rdflib.XSD.string)\n",
    "        ))       \n",
    "        \n",
    "nodes['institutes'] = {}\n",
    "for instStr in study_sheet['Contact institution*']:\n",
    "    inst = instStr.strip()\n",
    "    inst_hash = int(hashlib.sha1(inst.encode('utf-8')).hexdigest(), 16) % (10 ** 8)\n",
    "    if inst not in nodes['institutes']:\n",
    "        nodes['institutes'][inst] = URIRef('#institute/' + str(inst_hash)) #rdflib.BNode() \n",
    "        instParts = [x.strip() for x in inst.split(';')]\n",
    "        g.add((\n",
    "            nodes['institutes'][inst],\n",
    "            ppeo.hasName,\n",
    "            rdflib.Literal(instParts[0])\n",
    "        ))\n",
    "        g.add((\n",
    "            nodes['institutes'][inst],\n",
    "            rdf.type,\n",
    "            ppeo.institution\n",
    "        ))\n",
    "        inst_loc_hash = int(hashlib.sha1((inst + ' ' + 'location').encode('utf-8')).hexdigest(), 16) % (10 ** 8)                            \n",
    "        instLocNode = URIRef('#location/' + str(inst_loc_hash))  # rdflib.BNode() \n",
    "        g.add((\n",
    "            nodes['institutes'][inst],\n",
    "            ppeo.hasLocation,\n",
    "            instLocNode\n",
    "        ))\n",
    "        g.add((\n",
    "            instLocNode,\n",
    "            rdf.type,\n",
    "            ppeo.location\n",
    "        )) \n",
    "        g.add((\n",
    "            instLocNode,\n",
    "            ppeo.hasName,\n",
    "            rdflib.Literal(instParts[0])\n",
    "        ))\n",
    "        g.add((\n",
    "            instLocNode,\n",
    "            ppeo.hasAddress,\n",
    "            rdflib.Literal(instParts[1])\n",
    "        ))\n",
    "        if instParts[2] in nodes['countries']:\n",
    "            g.add((\n",
    "                instLocNode,\n",
    "                ppeo.hasCountry,\n",
    "                nodes['countries'][instParts[2]]\n",
    "            ))  \n",
    "        else:\n",
    "            print('Country ' + instParts[2] + ' for ' + instStr + ' not found: not implemented.')\n",
    "\n",
    "            \n",
    "for index, row in study_sheet.iterrows():\n",
    "    #print(row['Study unique ID*'], row['Study title*'])\n",
    "    \n",
    "    stuIDstp = row['Study unique ID*'].strip()\n",
    "    curStudyNode = nodes['studies'][stuIDstp]\n",
    "    curStudyID = make_safe(stuIDstp)\n",
    "    \n",
    "    g.add((\n",
    "        nodes['investigation'],\n",
    "        ppeo.hasPart,\n",
    "        curStudyNode\n",
    "    ))\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        rdf.type,\n",
    "        ppeo.study\n",
    "    ))\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasIdentifier,\n",
    "        rdflib.Literal(row['Study unique ID*'])\n",
    "    ))\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasName,\n",
    "        rdflib.Literal(row['Study title*'])\n",
    "    ))\n",
    "    \n",
    "    curElem = row['Study description']\n",
    "    if curElem and curElem.strip():\n",
    "        g.add((\n",
    "            curStudyNode,\n",
    "            ppeo.hasDescription,\n",
    "            rdflib.Literal(curElem)\n",
    "        ))\n",
    "    \n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasStartDateTime,\n",
    "        rdflib.Literal(row['Start date of study*'], datatype=rdflib.XSD.date)\n",
    "    ))\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasEndDateTime,\n",
    "        rdflib.Literal(row['End date of study'], datatype=rdflib.XSD.date)\n",
    "    ))\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasContactInstitution,\n",
    "        rdflib.Literal(row['Contact institution*'])\n",
    "    ))\n",
    "\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasLocation,\n",
    "        nodes['locations'][row['Experimental site name*'].strip()]\n",
    "    ))\n",
    "\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasDescriptionOfExperimentalDesign,\n",
    "        rdflib.Literal(row['Description of statistical design*'])\n",
    "    ))\n",
    "    \n",
    "    hierarchyText = rdflib.Literal(row['Observation unit level hierarchy']).strip()\n",
    "    if hierarchyText:\n",
    "        olhNode = URIRef('#observation_level_hierarchy/' + inv_ID + '_' + curStudyID + '_olh')  # rdflib.BNode() \n",
    "        nodes['obsLevelHierarchy'][stuIDstp]['hierarchy'] = olhNode\n",
    "        g.add((\n",
    "            curStudyNode,\n",
    "            ppeo.hasObservationUnitLevelHierarchy,\n",
    "            olhNode\n",
    "        ))\n",
    "        g.add((\n",
    "            olhNode,\n",
    "            rdf.type,\n",
    "            ppeo.observation_level_hierarchy\n",
    "        ))\n",
    "        \n",
    "        levels = [x.strip() for x in hierarchyText.split('>')]\n",
    "        levelNode =  URIRef('#observation_level/' + inv_ID + '_' + curStudyID + '_' + levels[0])  # rdflib.BNode()\n",
    "        nodes['obsLevelHierarchy'][stuIDstp][levels[0]] = levelNode\n",
    "        g.add((\n",
    "            levelNode,\n",
    "            ppeo.partOf,\n",
    "            olhNode\n",
    "        ))\n",
    "        g.add((\n",
    "            levelNode,\n",
    "            rdf.type,\n",
    "            ppeo.observation_level\n",
    "        ))\n",
    "        g.add((\n",
    "            levelNode,\n",
    "            ppeo.hasType,\n",
    "            rdflib.Literal(levels[0])\n",
    "        ))\n",
    "        previousLevel = levelNode\n",
    "        for L in levels[1:]:\n",
    "            levelNode = URIRef('#observation_level/' + inv_ID + '_' + curStudyID + '_' + L)  # rdflib.BNode() \n",
    "            nodes['obsLevelHierarchy'][stuIDstp][L] = levelNode\n",
    "            g.add((\n",
    "                levelNode,\n",
    "                ppeo.partOf,\n",
    "                olhNode\n",
    "            ))\n",
    "            g.add((\n",
    "                levelNode,\n",
    "                rdf.type,\n",
    "                ppeo.observation_level\n",
    "            ))\n",
    "            g.add((\n",
    "                levelNode,\n",
    "                ppeo.hasType,\n",
    "                rdflib.Literal(L)\n",
    "            ))\n",
    "            g.add((\n",
    "                levelNode,\n",
    "                ppeo.hasSuperlevel,\n",
    "                previousLevel\n",
    "            ))\n",
    "    \n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasObservationUnitDescription,\n",
    "        rdflib.Literal(row['Observation unit description*'])\n",
    "    ))\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasDescriptionOfGrowthFacility,\n",
    "        rdflib.Literal(row['Description of growth facility*'])\n",
    "    ))\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasTypeOfGrowthFacility,\n",
    "        rdflib.Literal(row['Type of growth facility'])\n",
    "    ))\n",
    "    g.add((\n",
    "        curStudyNode,\n",
    "        ppeo.hasCulturalPractices,\n",
    "        rdflib.Literal(row['Cultural practices'])\n",
    "    ))\n",
    "    \n",
    "\n",
    "# print(g.serialize(format='ttl').decode('utf8'))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "code_folding": [
     0,
     49
    ]
   },
   "outputs": [],
   "source": [
    "# Person\n",
    "nodes['persons'] = {}\n",
    "for person in person_sheet['Person name*']:\n",
    "    if not person: continue\n",
    "    nodes['persons'][person] = URIRef('#person/' + make_safe(person)) #rdflib.BNode()\n",
    "\n",
    "for index, row in person_sheet.iterrows():\n",
    "    if not (row['Person role*'] and row['Person name*']): break\n",
    "    curPersonNode = nodes['persons'][row['Person name*']]\n",
    "    person_roles = [x.strip() for x in row['Person role*'].split(';')]\n",
    "    for pr in person_roles:\n",
    "        #if not pr: break\n",
    "        curRoleNode = URIRef('#person_role/' + inv_ID + '_' + make_safe(curStudyID) + \n",
    "                             '_' + make_safe(pr)) # rdflib.BNode()\n",
    "        g.add((\n",
    "            nodes['investigation'],\n",
    "            ppeo.hasPersonWithRole,\n",
    "            curRoleNode\n",
    "        ))\n",
    "        \n",
    "        g.add((\n",
    "            curRoleNode,\n",
    "            ppeo.hasName,\n",
    "            rdflib.Literal(pr)\n",
    "        ))\n",
    "\n",
    "        g.add((\n",
    "            curRoleNode,\n",
    "            ppeo.isRoleOf,\n",
    "            curPersonNode,\n",
    "        ))\n",
    "\n",
    "    g.add((\n",
    "        curPersonNode,\n",
    "        rdf.type,\n",
    "        ppeo.person\n",
    "    ))\n",
    "    g.add((\n",
    "        curPersonNode,\n",
    "        ppeo.hasName,\n",
    "        rdflib.Literal(row['Person name*'])\n",
    "    ))\n",
    "    g.add((\n",
    "        curPersonNode,\n",
    "        ppeo.hasEmail,\n",
    "        rdflib.Literal(row['Person email'])\n",
    "    ))\n",
    "    \n",
    "    curElem = row['Person ID']\n",
    "    if curElem and curElem.strip():\n",
    "        g.add((\n",
    "            curPersonNode,\n",
    "            ppeo.hasIdentifier,\n",
    "            rdflib.Literal(curElem)\n",
    "        ))\n",
    "\n",
    "    g.add((\n",
    "        curPersonNode,\n",
    "        ppeo.hasAffiliation,\n",
    "        rdflib.Literal(row['Person affiliation*'])\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Data file\n",
    "\n",
    "nodes['datafiles'] = {}\n",
    "for index, row in datafile_sheet.iterrows():\n",
    "    filename = row['Data file link*'].strip()\n",
    "    fileNode = URIRef('#data_file/' + inv_ID + '_' + make_safe(filename))  # rdflib.BNode() \n",
    "    nodes['datafiles'][filename] = fileNode\n",
    "    studyNode = nodes['studies'][row['Study unique ID*'].strip()]\n",
    "\n",
    "    g.add((\n",
    "        studyNode,\n",
    "        ppeo.hasDataFile,\n",
    "        fileNode\n",
    "    ))\n",
    "    g.add((\n",
    "        fileNode,\n",
    "        rdf.type,\n",
    "        ppeo.data_file\n",
    "    ))\n",
    "    g.add((\n",
    "        fileNode,\n",
    "        ppeo.hasDigitalLocation,\n",
    "        rdflib.Literal(filename)\n",
    "    ))\n",
    "    g.add((\n",
    "        fileNode,\n",
    "        ppeo.hasDescription,\n",
    "        rdflib.Literal(row['Data file description*'].strip())\n",
    "    ))\n",
    "    g.add((\n",
    "        fileNode,\n",
    "        ppeo.hasVersion,\n",
    "        rdflib.Literal(row['Data file version*'])\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Biological material\n",
    "\n",
    "nodes['materials'] = {}\n",
    "#for study in study_sheet['Study unique ID*']:\n",
    "#    nodes['studies'][study] = rdflib.BNode()\n",
    "# pp.pprint(nodes)\n",
    "\n",
    "nodes['bm'] = {}\n",
    "for bm in bm_sheet['Biological material ID*']:\n",
    "    nodes['bm'][bm] = URIRef('#biological_material/' + inv_ID + '_' + make_safe(bm.strip()))  # rdflib.BNode()\n",
    "\n",
    "nodes['ms'] = {}\n",
    "for index, row in bm_sheet.iterrows():\n",
    "\n",
    "    curBmNode = nodes['bm'][row['Biological material ID*']]\n",
    "    studies_for_bm = [x.strip() for x in row['Study unique ID*'].split(';')]\n",
    "\n",
    "    for s in studies_for_bm:\n",
    "        g.add((\n",
    "            nodes['studies'][s],\n",
    "            ppeo.hasBiologicalMaterial,\n",
    "            curBmNode\n",
    "        ))\n",
    "    g.add((\n",
    "        curBmNode,\n",
    "        rdf.type,\n",
    "        ppeo.biological_material\n",
    "    ))\n",
    "    g.add((\n",
    "        curBmNode,\n",
    "        ppeo.hasIdentifier,\n",
    "        rdflib.Literal(row['Biological material ID*'])\n",
    "    ))\n",
    "    g.add((\n",
    "        curBmNode,\n",
    "        ppeo.hasTaxonIdentifier,\n",
    "        rdflib.Literal(row['Organism*'])\n",
    "    ))\n",
    "    g.add((\n",
    "        curBmNode,\n",
    "        ppeo.hasGenus,\n",
    "        rdflib.Literal(row['Genus'])\n",
    "    ))\n",
    "    g.add((\n",
    "        curBmNode,\n",
    "        ppeo.hasSpecies,\n",
    "        rdflib.Literal(row['Species'])\n",
    "    ))\n",
    "    \n",
    "    curElem = row['Material source ID (Holding institute/stock centre, accession)']\n",
    "    if curElem and curElem.strip():\n",
    "        if curElem not in nodes['ms']:\n",
    "            nodes['ms'][curElem] = URIRef('material_source/' + inv_ID + '_' + \n",
    "                                          make_safe(curElem.strip()))  # rdflib.BNode()\n",
    "            curElemNode = nodes['ms'][curElem]\n",
    "            g.add((\n",
    "               curElemNode,\n",
    "                rdf.type,\n",
    "                ppeo.material_source\n",
    "            ))\n",
    "            g.add((\n",
    "                curElemNode,\n",
    "                ppeo.hasIdentifier,\n",
    "                rdflib.Literal(curElem)\n",
    "            ))\n",
    "            g.add((\n",
    "                curElemNode,\n",
    "                ppeo.hasDOI,\n",
    "                rdflib.Literal(row['Material source DOI'])\n",
    "            ))\n",
    "            g.add((\n",
    "                curElemNode,\n",
    "                ppeo.hasDescription,\n",
    "                rdflib.Literal(row['Material source description'])\n",
    "            ))\n",
    "        curElemNode = nodes['ms'][curElem]    \n",
    "        g.add((\n",
    "            curBmNode,\n",
    "            ppeo.hasMaterialSource,\n",
    "            curElemNode\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Observation unit\n",
    "\n",
    "nodes['ou'] = {}\n",
    "for ou in ou_sheet['Observation unit ID*']:\n",
    "    nodes['ou'][ou] = URIRef('#observation_unit/' + inv_ID + '_' + make_safe(ou.strip()))  # rdflib.BNode()\n",
    "\n",
    "sd_counter = 0\n",
    "for index, row in ou_sheet.iterrows():\n",
    "\n",
    "    curOuNode = nodes['ou'][row['Observation unit ID*']]\n",
    "    stuIDstp = row['Study unique ID*'].strip()\n",
    "    curStudyID = make_safe(stuIDstp)\n",
    "    g.add((\n",
    "        curOuNode,\n",
    "        rdf.type,\n",
    "        ppeo.observation_unit\n",
    "    ))\n",
    "    g.add((\n",
    "        curOuNode,\n",
    "        ppeo.hasIdentifier,\n",
    "        rdflib.Literal(row['Observation unit ID*'])       \n",
    "    ))\n",
    "    g.add((\n",
    "        curOuNode,\n",
    "        ppeo.partOf,\n",
    "        nodes['studies'][stuIDstp]\n",
    "    ))\n",
    "    g.add((\n",
    "        nodes['studies'][row['Study unique ID*'].strip()],\n",
    "        ppeo.hasPart,\n",
    "        curOuNode        \n",
    "    ))\n",
    "    \n",
    "    if '#' in row['Biological Material ID*']:\n",
    "        print(row['Observation unit ID*'])\n",
    "    g.add((\n",
    "        curOuNode,\n",
    "        ppeo.hasBiologicalMaterial,\n",
    "        nodes['bm'][row['Biological Material ID*'].strip()]\n",
    "    ))\n",
    "    \n",
    "    ouType = row['Observation unit type*'].strip()\n",
    "    if ouType in nodes['obsLevelHierarchy'][stuIDstp]:\n",
    "        typeNode = nodes['obsLevelHierarchy'][stuIDstp][ouType]\n",
    "        g.add((\n",
    "            curOuNode,\n",
    "            ppeo.hasType,\n",
    "            typeNode\n",
    "        ))\n",
    "    else: \n",
    "        print('thisError: observation unit type ' + ouType + ' not found in study ' + stuIDstp)\n",
    "\n",
    "    \n",
    "    if row['Spatial distribution']:\n",
    "        #print('found SD: ' + row['Spatial distribution'] )\n",
    "        spatDistParts = row['Spatial distribution'].split(';')\n",
    "        for sdPart in spatDistParts:\n",
    "            parts = [x.strip() for x in sdPart.split(':')]\n",
    "            #print(parts)\n",
    "            if len(parts) < 2: \n",
    "                continue\n",
    "                print('skipped spatial distribution')\n",
    "            sdType, sdVal = parts[0], parts[1]\n",
    "            #print(sdType, sdVal)\n",
    "            \n",
    "            if sdType in nodes['obsLevelHierarchy'][stuIDstp]:\n",
    "                typeNode = nodes['obsLevelHierarchy'][stuIDstp][sdType]\n",
    "            elif sdType in nodes['spatDist'][stuIDstp]:\n",
    "                typeNode = nodes['spatDist'][stuIDstp][sdType]\n",
    "            else: \n",
    "                typeNode = URIRef('#spatial_distribition_type/' + inv_ID + '_' + \n",
    "                                  curStudyID + make_safe(sdType))  # rdflib.BNode()\n",
    "                nodes['spatDist'][stuIDstp][sdType] = typeNode\n",
    "                g.add((\n",
    "                    typeNode,\n",
    "                    rdf.type,\n",
    "                    ppeo.spatial_distribution_type\n",
    "                ))\n",
    "                g.add((\n",
    "                    typeNode,\n",
    "                    ppeo.hasType,\n",
    "                    rdflib.Literal(sdType)\n",
    "                ))\n",
    "                \n",
    "            curSdNode = URIRef('#spatial_distribution/' + inv_ID + '_' + \n",
    "                                curStudyID + '_' + make_safe(row['Observation unit ID*']) + '_' + str(sd_counter))  # rdflib.BNode()\n",
    "            sd_counter += 1\n",
    "            \n",
    "            g.add((\n",
    "                curOuNode,\n",
    "                ppeo.hasSpatialDistribution,\n",
    "                curSdNode\n",
    "            ))\n",
    "            g.add((\n",
    "                curSdNode,\n",
    "                ppeo.hasSpatialDistributionType,\n",
    "                typeNode\n",
    "            ))\n",
    "            g.add((\n",
    "                curSdNode,\n",
    "                ppeo.hasValue,\n",
    "                rdflib.Literal(sdVal)\n",
    "            ))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Observed variables\n",
    "\n",
    "inv_ID = nodes['investigation_ID']\n",
    "nodes['ov'] = {}\n",
    "nodes['trait'] = {}\n",
    "nodes['method'] = {}\n",
    "nodes['scale'] = {}\n",
    "for ov in ov_sheet['Variable ID*']:\n",
    "    nodes['ov'][ov] = URIRef('#variable/' + inv_ID + '_' + make_safe(ov.strip()))  # rdflib.BNode()\n",
    "\n",
    "for index, row in ov_sheet.iterrows():\n",
    "\n",
    "    curOvNode = nodes['ov'][row['Variable ID*']]\n",
    "    studies_for_ov = [x.strip() for x in row['Study unique ID*'].split(';')]\n",
    "\n",
    "    for s in studies_for_ov:\n",
    "        g.add((\n",
    "            nodes['studies'][s],\n",
    "            ppeo.hasPart,\n",
    "            curOvNode\n",
    "        ))\n",
    "    g.add((\n",
    "        curOvNode,\n",
    "        rdf.type,\n",
    "        ppeo.observed_variable\n",
    "    ))\n",
    "    g.add((\n",
    "        curOvNode,\n",
    "        ppeo.hasIdentifier,\n",
    "        rdflib.Literal(row['Variable ID*'])\n",
    "    ))\n",
    "    \n",
    "    t = row['Trait*'].strip()\n",
    "    tAc = ''\n",
    "    if row['Trait accession number']:\n",
    "        tAc = row['Trait accession number'].strip()\n",
    "    if t in nodes['trait']:\n",
    "        tNode = nodes['trait'][t]\n",
    "    else:\n",
    "        tHash = int(hashlib.sha1(t.encode('utf-8')).hexdigest(), 16) % (10 ** 8)\n",
    "        tNode = URIRef('#trait/' + inv_ID + '_' + str(tHash))  # rdflib.BNode()\n",
    "        g.add((\n",
    "            tNode,\n",
    "            ppeo.hasName,\n",
    "            rdflib.Literal(t)\n",
    "        ))\n",
    "        if tAc:\n",
    "            g.add((\n",
    "                tNode,\n",
    "                ppeo.hasTraitAccessionNumber,\n",
    "                rdflib.Literal(tAc)\n",
    "            ))\n",
    "        \n",
    "    g.add((\n",
    "        curOvNode,\n",
    "        ppeo.hasTrait,\n",
    "        tNode\n",
    "    ))\n",
    "\n",
    "    m = row['Method*'].strip()\n",
    "    mAc = ''\n",
    "    if row['Method accession number']:\n",
    "        mAc = row['Method accession number'].strip()\n",
    "    if m in nodes['method']:\n",
    "        mNode = nodes['method'][m]\n",
    "    else:\n",
    "        mHash = int(hashlib.sha1(m.encode('utf-8')).hexdigest(), 16) % (10 ** 8)\n",
    "        mNode = URIRef('#method/' + inv_ID + '_' + str(mHash))  # rdflib.BNode()\n",
    "        nodes['method'][m] = mNode\n",
    "        g.add((\n",
    "            mNode,\n",
    "            ppeo.hasName,\n",
    "            rdflib.Literal(m)\n",
    "        ))\n",
    "        if mAc:\n",
    "            g.add((\n",
    "                mNode,\n",
    "                ppeo.hasMethodAccessionNumber,\n",
    "                rdflib.Literal(mAc)\n",
    "            ))\n",
    "        \n",
    "    g.add((\n",
    "        curOvNode,\n",
    "        ppeo.hasMethod,\n",
    "        mNode\n",
    "    ))    \n",
    "    \n",
    "    \n",
    "    s = row['Scale*'].strip()\n",
    "    sAc = ''\n",
    "    if row['Scale accession number']:\n",
    "        sAc = row['Scale accession number'].strip()\n",
    "    if s in nodes['scale']:\n",
    "        sNode = nodes['scale'][s]\n",
    "    else:\n",
    "        sHash = int(hashlib.sha1(s.encode('utf-8')).hexdigest(), 16) % (10 ** 8)\n",
    "        sNode = URIRef('#scale/' + inv_ID + '_' + str(sHash))  # rdflib.BNode()\n",
    "        nodes['scale'][s] = sNode\n",
    "        g.add((\n",
    "            sNode,\n",
    "            ppeo.hasName,\n",
    "            rdflib.Literal(s)\n",
    "        ))\n",
    "        if sAc:\n",
    "            g.add((\n",
    "                sNode,\n",
    "                ppeo.hasMethodAccessionNumber,\n",
    "                rdflib.Literal(sAc)\n",
    "            ))\n",
    "        \n",
    "    g.add((\n",
    "        curOvNode,\n",
    "        ppeo.hasScale,\n",
    "        sNode\n",
    "    )) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# print(g.serialize(format='ttl').decode('utf8')) \n",
    "g.serialize(destination='data-generated' + os.sep + 'miappe_metadata.ttl', \n",
    "            base='http://localhost:3030/wur/fair-example', \n",
    "            format='turtle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform actual data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Netherlands 1999 data to ttl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1999NL =  pd.read_csv(data_or + 'data_1999NL.csv', sep='\\t')\n",
    "df_1999NL = df_1999NL.where(pd.notnull(df_1999NL), None)\n",
    "df_1999NL['Date'] = pd.to_datetime(df_1999NL['Date']).dt.date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Are there any missing values?  True\n",
      "Are there any missing values in the Date column?  False\n",
      "How many missing values are there overall?  1015\n",
      "Missing value percentage: 35.1%\n"
     ]
    }
   ],
   "source": [
    "# look at data info\n",
    "df_1999NL;\n",
    "df_1999NL.dtypes;\n",
    "# df_1999NL.info();\n",
    "# with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also\n",
    "#    print(df_1999NL)\n",
    "'ok'\n",
    "# examining missing values\n",
    "print('Are there any missing values? ', df_1999NL.isnull().values.any())\n",
    "print('Are there any missing values in the Date column? ', df_1999NL['Date'].isnull().values.any())\n",
    "# print('How many missing values are there per column?\\n', df_1999NL.isnull().sum())\n",
    "print('How many missing values are there overall? ', df_1999NL.isnull().sum().sum())\n",
    "total_cells = len(df_1999NL.columns) * len(df_1999NL)\n",
    "print('Missing value percentage: ' + str( \n",
    "      round(df_1999NL.isnull().sum().sum() * 100 / total_cells,2)) + '%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# g.remove((None, None, None))\n",
    "g = parseStudyData('data_1999NL.csv', df_1999NL, nodes)\n",
    "#print(g_1999NL.serialize(format='ttl').decode('utf8'))\n",
    "g.serialize(destination=data_gen + 'data_1999NL.ttl', format='ttl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Venezuela 2003 data to ttl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Observation unit ID</th>\n",
       "      <th>Date</th>\n",
       "      <th>TubN_average_per_genotype</th>\n",
       "      <th>TubW_average_per_genotype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WUR:CE-VEN03-CE011</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>18.5</td>\n",
       "      <td>142.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WUR:CE-VEN03-CE032</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>8.0</td>\n",
       "      <td>237.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WUR:CE-VEN03-CE047</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>21.0</td>\n",
       "      <td>230.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WUR:CE-VEN03-CE067</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>11.5</td>\n",
       "      <td>62.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WUR:CE-VEN03-CE070</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>7.0</td>\n",
       "      <td>125.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>WUR:CE-VEN03-CE100</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>8.0</td>\n",
       "      <td>127.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>WUR:CE-VEN03-CE102</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>16.0</td>\n",
       "      <td>176.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>WUR:CE-VEN03-CE110</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>9.5</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>WUR:CE-VEN03-CE155</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>27.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>WUR:CE-VEN03-CE159</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>22.0</td>\n",
       "      <td>272.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>WUR:CE-VEN03-CE165</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>17.0</td>\n",
       "      <td>339.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>WUR:CE-VEN03-CE195</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>10.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>WUR:CE-VEN03-CE200</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>19.0</td>\n",
       "      <td>72.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>WUR:CE-VEN03-CE202</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>9.0</td>\n",
       "      <td>40.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>WUR:CE-VEN03-CE207</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>26.0</td>\n",
       "      <td>220.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>WUR:CE-VEN03-CE210</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>23.0</td>\n",
       "      <td>177.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>WUR:CE-VEN03-CE233</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>18.0</td>\n",
       "      <td>74.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>WUR:CE-VEN03-CE252</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>12.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>WUR:CE-VEN03-CE276</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>10.5</td>\n",
       "      <td>96.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>WUR:CE-VEN03-CE298</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>17.0</td>\n",
       "      <td>75.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>WUR:CE-VEN03-CE601</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>11.0</td>\n",
       "      <td>52.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>WUR:CE-VEN03-CE602</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>14.5</td>\n",
       "      <td>76.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>WUR:CE-VEN03-CE603</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>15.0</td>\n",
       "      <td>110.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>WUR:CE-VEN03-CE604</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>8.5</td>\n",
       "      <td>85.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>WUR:CE-VEN03-CE605</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>12.0</td>\n",
       "      <td>174.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>WUR:CE-VEN03-CE607</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>11.0</td>\n",
       "      <td>115.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>WUR:CE-VEN03-CE609</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>14.0</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>WUR:CE-VEN03-CE611</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>7.0</td>\n",
       "      <td>98.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>WUR:CE-VEN03-CE612</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>24.5</td>\n",
       "      <td>137.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>WUR:CE-VEN03-CE613</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>13.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>WUR:CE-VEN03-CE705</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>8.0</td>\n",
       "      <td>27.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>WUR:CE-VEN03-CE706</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>36.5</td>\n",
       "      <td>76.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>WUR:CE-VEN03-CE707</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>11.5</td>\n",
       "      <td>71.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>WUR:CE-VEN03-CE710</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>32.0</td>\n",
       "      <td>168.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>WUR:CE-VEN03-CE713</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>10.0</td>\n",
       "      <td>975.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>WUR:CE-VEN03-CE715</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>3.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>WUR:CE-VEN03-CE719</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>9.0</td>\n",
       "      <td>77.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>WUR:CE-VEN03-CE721</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>17.0</td>\n",
       "      <td>103.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>WUR:CE-VEN03-CE729</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>9.0</td>\n",
       "      <td>145.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>WUR:CE-VEN03-CE731</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>16.0</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>WUR:CE-VEN03-CE734</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>10.5</td>\n",
       "      <td>62.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>WUR:CE-VEN03-CE740</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>17.0</td>\n",
       "      <td>117.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>WUR:CE-VEN03-CE742</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>13.0</td>\n",
       "      <td>76.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>WUR:CE-VEN03-CE744</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>36.0</td>\n",
       "      <td>108.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>WUR:CE-VEN03-CE746</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>39.5</td>\n",
       "      <td>150.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>WUR:CE-VEN03-CE748</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>18.0</td>\n",
       "      <td>300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>WUR:CE-VEN03-CE750</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>11.5</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>WUR:CE-VEN03-CE754</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>9.5</td>\n",
       "      <td>51.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>WUR:CE-VEN03-CE755</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>13.0</td>\n",
       "      <td>300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>WUR:CE-VEN03-CE756</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>39.0</td>\n",
       "      <td>177.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>WUR:CE-VEN03-CE758</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>11.0</td>\n",
       "      <td>35.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>WUR:CE-VEN03-CE759</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>15.0</td>\n",
       "      <td>126.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>WUR:CE-VEN03-CE760</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>25.0</td>\n",
       "      <td>156.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>WUR:CE-VEN03-CE761</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>3.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>WUR:CE-VEN03-CE762</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>15.0</td>\n",
       "      <td>128.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>WUR:CE-VEN03-CE763</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>19.0</td>\n",
       "      <td>55.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>WUR:CE-VEN03-CE765</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>21.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>WUR:CE-VEN03-CE766</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>7.0</td>\n",
       "      <td>67.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>WUR:CE-VEN03-CE776</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>11.0</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>WUR:CE-VEN03-CE788</td>\n",
       "      <td>2003-11-30</td>\n",
       "      <td>7.0</td>\n",
       "      <td>48.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>107 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Observation unit ID        Date  TubN_average_per_genotype  \\\n",
       "0    WUR:CE-VEN03-CE011  2003-11-30                       18.5   \n",
       "1    WUR:CE-VEN03-CE032  2003-11-30                        8.0   \n",
       "2    WUR:CE-VEN03-CE047  2003-11-30                       21.0   \n",
       "3    WUR:CE-VEN03-CE067  2003-11-30                       11.5   \n",
       "4    WUR:CE-VEN03-CE070  2003-11-30                        7.0   \n",
       "5    WUR:CE-VEN03-CE100  2003-11-30                        8.0   \n",
       "6    WUR:CE-VEN03-CE102  2003-11-30                       16.0   \n",
       "7    WUR:CE-VEN03-CE110  2003-11-30                        9.5   \n",
       "8    WUR:CE-VEN03-CE155  2003-11-30                       27.0   \n",
       "9    WUR:CE-VEN03-CE159  2003-11-30                       22.0   \n",
       "10   WUR:CE-VEN03-CE165  2003-11-30                       17.0   \n",
       "11   WUR:CE-VEN03-CE195  2003-11-30                       10.0   \n",
       "12   WUR:CE-VEN03-CE200  2003-11-30                       19.0   \n",
       "13   WUR:CE-VEN03-CE202  2003-11-30                        9.0   \n",
       "14   WUR:CE-VEN03-CE207  2003-11-30                       26.0   \n",
       "15   WUR:CE-VEN03-CE210  2003-11-30                       23.0   \n",
       "16   WUR:CE-VEN03-CE233  2003-11-30                       18.0   \n",
       "17   WUR:CE-VEN03-CE252  2003-11-30                       12.0   \n",
       "18   WUR:CE-VEN03-CE276  2003-11-30                       10.5   \n",
       "19   WUR:CE-VEN03-CE298  2003-11-30                       17.0   \n",
       "20   WUR:CE-VEN03-CE601  2003-11-30                       11.0   \n",
       "21   WUR:CE-VEN03-CE602  2003-11-30                       14.5   \n",
       "22   WUR:CE-VEN03-CE603  2003-11-30                       15.0   \n",
       "23   WUR:CE-VEN03-CE604  2003-11-30                        8.5   \n",
       "24   WUR:CE-VEN03-CE605  2003-11-30                       12.0   \n",
       "25   WUR:CE-VEN03-CE607  2003-11-30                       11.0   \n",
       "26   WUR:CE-VEN03-CE609  2003-11-30                       14.0   \n",
       "27   WUR:CE-VEN03-CE611  2003-11-30                        7.0   \n",
       "28   WUR:CE-VEN03-CE612  2003-11-30                       24.5   \n",
       "29   WUR:CE-VEN03-CE613  2003-11-30                       13.0   \n",
       "..                  ...         ...                        ...   \n",
       "77   WUR:CE-VEN03-CE705  2003-11-30                        8.0   \n",
       "78   WUR:CE-VEN03-CE706  2003-11-30                       36.5   \n",
       "79   WUR:CE-VEN03-CE707  2003-11-30                       11.5   \n",
       "80   WUR:CE-VEN03-CE710  2003-11-30                       32.0   \n",
       "81   WUR:CE-VEN03-CE713  2003-11-30                       10.0   \n",
       "82   WUR:CE-VEN03-CE715  2003-11-30                        3.0   \n",
       "83   WUR:CE-VEN03-CE719  2003-11-30                        9.0   \n",
       "84   WUR:CE-VEN03-CE721  2003-11-30                       17.0   \n",
       "85   WUR:CE-VEN03-CE729  2003-11-30                        9.0   \n",
       "86   WUR:CE-VEN03-CE731  2003-11-30                       16.0   \n",
       "87   WUR:CE-VEN03-CE734  2003-11-30                       10.5   \n",
       "88   WUR:CE-VEN03-CE740  2003-11-30                       17.0   \n",
       "89   WUR:CE-VEN03-CE742  2003-11-30                       13.0   \n",
       "90   WUR:CE-VEN03-CE744  2003-11-30                       36.0   \n",
       "91   WUR:CE-VEN03-CE746  2003-11-30                       39.5   \n",
       "92   WUR:CE-VEN03-CE748  2003-11-30                       18.0   \n",
       "93   WUR:CE-VEN03-CE750  2003-11-30                       11.5   \n",
       "94   WUR:CE-VEN03-CE754  2003-11-30                        9.5   \n",
       "95   WUR:CE-VEN03-CE755  2003-11-30                       13.0   \n",
       "96   WUR:CE-VEN03-CE756  2003-11-30                       39.0   \n",
       "97   WUR:CE-VEN03-CE758  2003-11-30                       11.0   \n",
       "98   WUR:CE-VEN03-CE759  2003-11-30                       15.0   \n",
       "99   WUR:CE-VEN03-CE760  2003-11-30                       25.0   \n",
       "100  WUR:CE-VEN03-CE761  2003-11-30                        3.0   \n",
       "101  WUR:CE-VEN03-CE762  2003-11-30                       15.0   \n",
       "102  WUR:CE-VEN03-CE763  2003-11-30                       19.0   \n",
       "103  WUR:CE-VEN03-CE765  2003-11-30                       21.0   \n",
       "104  WUR:CE-VEN03-CE766  2003-11-30                        7.0   \n",
       "105  WUR:CE-VEN03-CE776  2003-11-30                       11.0   \n",
       "106  WUR:CE-VEN03-CE788  2003-11-30                        7.0   \n",
       "\n",
       "     TubW_average_per_genotype  \n",
       "0                        142.5  \n",
       "1                        237.5  \n",
       "2                        230.0  \n",
       "3                         62.5  \n",
       "4                        125.0  \n",
       "5                        127.5  \n",
       "6                        176.0  \n",
       "7                         56.0  \n",
       "8                        100.0  \n",
       "9                        272.5  \n",
       "10                       339.5  \n",
       "11                        50.0  \n",
       "12                        72.5  \n",
       "13                        40.5  \n",
       "14                       220.0  \n",
       "15                       177.0  \n",
       "16                        74.5  \n",
       "17                        27.0  \n",
       "18                        96.0  \n",
       "19                        75.0  \n",
       "20                        52.0  \n",
       "21                        76.0  \n",
       "22                       110.0  \n",
       "23                        85.0  \n",
       "24                       174.0  \n",
       "25                       115.0  \n",
       "26                       120.0  \n",
       "27                        98.0  \n",
       "28                       137.5  \n",
       "29                       100.0  \n",
       "..                         ...  \n",
       "77                        27.5  \n",
       "78                        76.5  \n",
       "79                        71.5  \n",
       "80                       168.5  \n",
       "81                       975.0  \n",
       "82                        50.0  \n",
       "83                        77.0  \n",
       "84                       103.0  \n",
       "85                       145.0  \n",
       "86                        15.0  \n",
       "87                        62.5  \n",
       "88                       117.5  \n",
       "89                        76.0  \n",
       "90                       108.0  \n",
       "91                       150.0  \n",
       "92                       300.0  \n",
       "93                       100.0  \n",
       "94                        51.5  \n",
       "95                       300.0  \n",
       "96                       177.0  \n",
       "97                        35.5  \n",
       "98                       126.0  \n",
       "99                       156.0  \n",
       "100                       12.0  \n",
       "101                      128.0  \n",
       "102                       55.5  \n",
       "103                       56.0  \n",
       "104                       67.0  \n",
       "105                       50.0  \n",
       "106                       48.5  \n",
       "\n",
       "[107 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2003VE =  pd.read_csv(data_or + 'data_2003VE.csv', sep='\\t')\n",
    "df_2003VE = df_2003VE.where(pd.notnull(df_2003VE), None)\n",
    "df_2003VE['Date']= pd.to_datetime(df_2003VE['Date']).dt.date\n",
    "df_2003VE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Are there any missing values?  False\n",
      "Are there any missing values in the Date column?  False\n",
      "How many missing values are there overall?  0\n",
      "Missing value percentage: 0.0%\n"
     ]
    }
   ],
   "source": [
    "# look at data info\n",
    "df_2003VE;\n",
    "df_2003VE.dtypes;\n",
    "# df_2003VE.info();\n",
    "# with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also\n",
    "#    print(df_1999NL)\n",
    "\n",
    "# examining missing values\n",
    "print('Are there any missing values? ', df_2003VE.isnull().values.any())\n",
    "print('Are there any missing values in the Date column? ',df_2003VE['Date'].isnull().values.any())\n",
    "# print('How many missing values are there per column?\\n', df_2003VE.isnull().sum())\n",
    "print('How many missing values are there overall? ', df_2003VE.isnull().sum().sum())\n",
    "total_cells = len(df_2003VE.columns) * len(df_2003VE)\n",
    "print('Missing value percentage: ' + str( \n",
    "      round(df_2003VE.isnull().sum().sum() * 100 / total_cells,2)) + '%')\n",
    "\n",
    "# g.remove((None, None, None))\n",
    "g = parseStudyData('data_2003VE.csv', df_2003VE, nodes)\n",
    "#print(g_1999NL.serialize(format='ttl').decode('utf8'))\n",
    "g.serialize(destination=data_gen + 'data_2003VE.ttl', format='ttl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ethiopia 2010 data to ttl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Observation unit ID</th>\n",
       "      <th>Date</th>\n",
       "      <th>TubN_LT20</th>\n",
       "      <th>TubW_LT20</th>\n",
       "      <th>TubN_GE20</th>\n",
       "      <th>TubW_GE20</th>\n",
       "      <th>TubN_total_per_plant</th>\n",
       "      <th>TubW_total_per_plant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WUR:CE-ETH10-1-1-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>10</td>\n",
       "      <td>140</td>\n",
       "      <td>10</td>\n",
       "      <td>410</td>\n",
       "      <td>20</td>\n",
       "      <td>550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WUR:CE-ETH10-1-1-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>2</td>\n",
       "      <td>46</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>5</td>\n",
       "      <td>191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WUR:CE-ETH10-1-1-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>5</td>\n",
       "      <td>75</td>\n",
       "      <td>9</td>\n",
       "      <td>460</td>\n",
       "      <td>14</td>\n",
       "      <td>535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WUR:CE-ETH10-1-1-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>3</td>\n",
       "      <td>65</td>\n",
       "      <td>4</td>\n",
       "      <td>185</td>\n",
       "      <td>7</td>\n",
       "      <td>250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WUR:CE-ETH10-1-2-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>5</td>\n",
       "      <td>86</td>\n",
       "      <td>6</td>\n",
       "      <td>255</td>\n",
       "      <td>11</td>\n",
       "      <td>341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>WUR:CE-ETH10-1-2-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>7</td>\n",
       "      <td>80</td>\n",
       "      <td>16</td>\n",
       "      <td>705</td>\n",
       "      <td>23</td>\n",
       "      <td>785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>WUR:CE-ETH10-1-2-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>10</td>\n",
       "      <td>143</td>\n",
       "      <td>9</td>\n",
       "      <td>340</td>\n",
       "      <td>19</td>\n",
       "      <td>483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>WUR:CE-ETH10-1-2-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>4</td>\n",
       "      <td>85</td>\n",
       "      <td>4</td>\n",
       "      <td>280</td>\n",
       "      <td>8</td>\n",
       "      <td>365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>WUR:CE-ETH10-1-3-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>16</td>\n",
       "      <td>104</td>\n",
       "      <td>9</td>\n",
       "      <td>305</td>\n",
       "      <td>25</td>\n",
       "      <td>409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>WUR:CE-ETH10-1-3-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>11</td>\n",
       "      <td>110</td>\n",
       "      <td>3</td>\n",
       "      <td>95</td>\n",
       "      <td>14</td>\n",
       "      <td>205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>WUR:CE-ETH10-1-3-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>8</td>\n",
       "      <td>90</td>\n",
       "      <td>10</td>\n",
       "      <td>400</td>\n",
       "      <td>18</td>\n",
       "      <td>490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>WUR:CE-ETH10-1-3-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>19</td>\n",
       "      <td>225</td>\n",
       "      <td>7</td>\n",
       "      <td>270</td>\n",
       "      <td>26</td>\n",
       "      <td>495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>WUR:CE-ETH10-1-4-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>14</td>\n",
       "      <td>146</td>\n",
       "      <td>12</td>\n",
       "      <td>425</td>\n",
       "      <td>26</td>\n",
       "      <td>571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>WUR:CE-ETH10-1-4-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>15</td>\n",
       "      <td>110</td>\n",
       "      <td>12</td>\n",
       "      <td>526</td>\n",
       "      <td>27</td>\n",
       "      <td>636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>WUR:CE-ETH10-1-4-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>9</td>\n",
       "      <td>105</td>\n",
       "      <td>15</td>\n",
       "      <td>505</td>\n",
       "      <td>24</td>\n",
       "      <td>610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>WUR:CE-ETH10-1-4-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>6</td>\n",
       "      <td>85</td>\n",
       "      <td>10</td>\n",
       "      <td>420</td>\n",
       "      <td>16</td>\n",
       "      <td>505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>WUR:CE-ETH10-1-5-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>175</td>\n",
       "      <td>7</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>WUR:CE-ETH10-1-5-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>7</td>\n",
       "      <td>95</td>\n",
       "      <td>5</td>\n",
       "      <td>260</td>\n",
       "      <td>12</td>\n",
       "      <td>355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>WUR:CE-ETH10-1-5-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>6</td>\n",
       "      <td>95</td>\n",
       "      <td>5</td>\n",
       "      <td>155</td>\n",
       "      <td>11</td>\n",
       "      <td>250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>WUR:CE-ETH10-1-5-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>12</td>\n",
       "      <td>130</td>\n",
       "      <td>4</td>\n",
       "      <td>105</td>\n",
       "      <td>16</td>\n",
       "      <td>235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>WUR:CE-ETH10-1-6-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>11</td>\n",
       "      <td>560</td>\n",
       "      <td>13</td>\n",
       "      <td>585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>WUR:CE-ETH10-1-6-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>11</td>\n",
       "      <td>555</td>\n",
       "      <td>11</td>\n",
       "      <td>555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>WUR:CE-ETH10-1-6-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>140</td>\n",
       "      <td>3</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>WUR:CE-ETH10-1-6-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>110</td>\n",
       "      <td>4</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>WUR:CE-ETH10-1-7-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>22</td>\n",
       "      <td>110</td>\n",
       "      <td>7</td>\n",
       "      <td>185</td>\n",
       "      <td>29</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>WUR:CE-ETH10-1-7-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>48</td>\n",
       "      <td>190</td>\n",
       "      <td>19</td>\n",
       "      <td>560</td>\n",
       "      <td>67</td>\n",
       "      <td>750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>WUR:CE-ETH10-1-7-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>21</td>\n",
       "      <td>115</td>\n",
       "      <td>16</td>\n",
       "      <td>415</td>\n",
       "      <td>37</td>\n",
       "      <td>530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>WUR:CE-ETH10-1-7-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>27</td>\n",
       "      <td>180</td>\n",
       "      <td>15</td>\n",
       "      <td>450</td>\n",
       "      <td>42</td>\n",
       "      <td>630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>WUR:CE-ETH10-1-8-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>520</td>\n",
       "      <td>13</td>\n",
       "      <td>530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>WUR:CE-ETH10-1-8-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>5</td>\n",
       "      <td>85</td>\n",
       "      <td>7</td>\n",
       "      <td>395</td>\n",
       "      <td>12</td>\n",
       "      <td>480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2274</th>\n",
       "      <td>WUR:CE-ETH10-3-185-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>8</td>\n",
       "      <td>210</td>\n",
       "      <td>14</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2275</th>\n",
       "      <td>WUR:CE-ETH10-3-185-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>7</td>\n",
       "      <td>180</td>\n",
       "      <td>12</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2276</th>\n",
       "      <td>WUR:CE-ETH10-3-186-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>205</td>\n",
       "      <td>6</td>\n",
       "      <td>215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2277</th>\n",
       "      <td>WUR:CE-ETH10-3-186-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2278</th>\n",
       "      <td>WUR:CE-ETH10-3-186-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>185</td>\n",
       "      <td>7</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2279</th>\n",
       "      <td>WUR:CE-ETH10-3-186-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>110</td>\n",
       "      <td>5</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2280</th>\n",
       "      <td>WUR:CE-ETH10-3-187-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>17</td>\n",
       "      <td>65</td>\n",
       "      <td>17</td>\n",
       "      <td>420</td>\n",
       "      <td>34</td>\n",
       "      <td>485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2281</th>\n",
       "      <td>WUR:CE-ETH10-3-187-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>16</td>\n",
       "      <td>60</td>\n",
       "      <td>13</td>\n",
       "      <td>325</td>\n",
       "      <td>29</td>\n",
       "      <td>385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2282</th>\n",
       "      <td>WUR:CE-ETH10-3-187-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>20</td>\n",
       "      <td>80</td>\n",
       "      <td>12</td>\n",
       "      <td>195</td>\n",
       "      <td>32</td>\n",
       "      <td>275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2283</th>\n",
       "      <td>WUR:CE-ETH10-3-187-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>9</td>\n",
       "      <td>40</td>\n",
       "      <td>18</td>\n",
       "      <td>415</td>\n",
       "      <td>27</td>\n",
       "      <td>455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2284</th>\n",
       "      <td>WUR:CE-ETH10-3-188-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>32</td>\n",
       "      <td>170</td>\n",
       "      <td>14</td>\n",
       "      <td>225</td>\n",
       "      <td>46</td>\n",
       "      <td>395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2285</th>\n",
       "      <td>WUR:CE-ETH10-3-188-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>12</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>115</td>\n",
       "      <td>18</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2286</th>\n",
       "      <td>WUR:CE-ETH10-3-188-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>13</td>\n",
       "      <td>105</td>\n",
       "      <td>16</td>\n",
       "      <td>300</td>\n",
       "      <td>29</td>\n",
       "      <td>405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2287</th>\n",
       "      <td>WUR:CE-ETH10-3-188-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>20</td>\n",
       "      <td>105</td>\n",
       "      <td>10</td>\n",
       "      <td>240</td>\n",
       "      <td>30</td>\n",
       "      <td>345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2288</th>\n",
       "      <td>WUR:CE-ETH10-3-189-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>215</td>\n",
       "      <td>14</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2289</th>\n",
       "      <td>WUR:CE-ETH10-3-189-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>195</td>\n",
       "      <td>8</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2290</th>\n",
       "      <td>WUR:CE-ETH10-3-189-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>4</td>\n",
       "      <td>105</td>\n",
       "      <td>4</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2291</th>\n",
       "      <td>WUR:CE-ETH10-3-189-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>160</td>\n",
       "      <td>13</td>\n",
       "      <td>170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2292</th>\n",
       "      <td>WUR:CE-ETH10-3-190-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>395</td>\n",
       "      <td>14</td>\n",
       "      <td>415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2293</th>\n",
       "      <td>WUR:CE-ETH10-3-190-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2294</th>\n",
       "      <td>WUR:CE-ETH10-3-190-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>320</td>\n",
       "      <td>8</td>\n",
       "      <td>325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2295</th>\n",
       "      <td>WUR:CE-ETH10-3-190-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>13</td>\n",
       "      <td>830</td>\n",
       "      <td>15</td>\n",
       "      <td>845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2296</th>\n",
       "      <td>WUR:CE-ETH10-3-191-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>12</td>\n",
       "      <td>55</td>\n",
       "      <td>10</td>\n",
       "      <td>160</td>\n",
       "      <td>22</td>\n",
       "      <td>215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2297</th>\n",
       "      <td>WUR:CE-ETH10-3-191-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>14</td>\n",
       "      <td>40</td>\n",
       "      <td>15</td>\n",
       "      <td>280</td>\n",
       "      <td>29</td>\n",
       "      <td>320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2298</th>\n",
       "      <td>WUR:CE-ETH10-3-191-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>14</td>\n",
       "      <td>105</td>\n",
       "      <td>14</td>\n",
       "      <td>195</td>\n",
       "      <td>28</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2299</th>\n",
       "      <td>WUR:CE-ETH10-3-191-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>95</td>\n",
       "      <td>12</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2300</th>\n",
       "      <td>WUR:CE-ETH10-3-192-1</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>6</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>7</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2301</th>\n",
       "      <td>WUR:CE-ETH10-3-192-2</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>6</td>\n",
       "      <td>215</td>\n",
       "      <td>15</td>\n",
       "      <td>245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2302</th>\n",
       "      <td>WUR:CE-ETH10-3-192-3</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>6</td>\n",
       "      <td>35</td>\n",
       "      <td>6</td>\n",
       "      <td>125</td>\n",
       "      <td>12</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2303</th>\n",
       "      <td>WUR:CE-ETH10-3-192-4</td>\n",
       "      <td>2010-12-01</td>\n",
       "      <td>4</td>\n",
       "      <td>30</td>\n",
       "      <td>3</td>\n",
       "      <td>105</td>\n",
       "      <td>7</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2304 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Observation unit ID        Date TubN_LT20 TubW_LT20 TubN_GE20  \\\n",
       "0       WUR:CE-ETH10-1-1-1  2010-12-01        10       140        10   \n",
       "1       WUR:CE-ETH10-1-1-2  2010-12-01         2        46         3   \n",
       "2       WUR:CE-ETH10-1-1-3  2010-12-01         5        75         9   \n",
       "3       WUR:CE-ETH10-1-1-4  2010-12-01         3        65         4   \n",
       "4       WUR:CE-ETH10-1-2-1  2010-12-01         5        86         6   \n",
       "5       WUR:CE-ETH10-1-2-2  2010-12-01         7        80        16   \n",
       "6       WUR:CE-ETH10-1-2-3  2010-12-01        10       143         9   \n",
       "7       WUR:CE-ETH10-1-2-4  2010-12-01         4        85         4   \n",
       "8       WUR:CE-ETH10-1-3-1  2010-12-01        16       104         9   \n",
       "9       WUR:CE-ETH10-1-3-2  2010-12-01        11       110         3   \n",
       "10      WUR:CE-ETH10-1-3-3  2010-12-01         8        90        10   \n",
       "11      WUR:CE-ETH10-1-3-4  2010-12-01        19       225         7   \n",
       "12      WUR:CE-ETH10-1-4-1  2010-12-01        14       146        12   \n",
       "13      WUR:CE-ETH10-1-4-2  2010-12-01        15       110        12   \n",
       "14      WUR:CE-ETH10-1-4-3  2010-12-01         9       105        15   \n",
       "15      WUR:CE-ETH10-1-4-4  2010-12-01         6        85        10   \n",
       "16      WUR:CE-ETH10-1-5-1  2010-12-01         2        15         5   \n",
       "17      WUR:CE-ETH10-1-5-2  2010-12-01         7        95         5   \n",
       "18      WUR:CE-ETH10-1-5-3  2010-12-01         6        95         5   \n",
       "19      WUR:CE-ETH10-1-5-4  2010-12-01        12       130         4   \n",
       "20      WUR:CE-ETH10-1-6-1  2010-12-01         2        25        11   \n",
       "21      WUR:CE-ETH10-1-6-2  2010-12-01      None      None        11   \n",
       "22      WUR:CE-ETH10-1-6-3  2010-12-01      None      None         3   \n",
       "23      WUR:CE-ETH10-1-6-4  2010-12-01         1        20         3   \n",
       "24      WUR:CE-ETH10-1-7-1  2010-12-01        22       110         7   \n",
       "25      WUR:CE-ETH10-1-7-2  2010-12-01        48       190        19   \n",
       "26      WUR:CE-ETH10-1-7-3  2010-12-01        21       115        16   \n",
       "27      WUR:CE-ETH10-1-7-4  2010-12-01        27       180        15   \n",
       "28      WUR:CE-ETH10-1-8-1  2010-12-01         1        10        12   \n",
       "29      WUR:CE-ETH10-1-8-2  2010-12-01         5        85         7   \n",
       "...                    ...         ...       ...       ...       ...   \n",
       "2274  WUR:CE-ETH10-3-185-3  2010-12-01         6        15         8   \n",
       "2275  WUR:CE-ETH10-3-185-4  2010-12-01         5        20         7   \n",
       "2276  WUR:CE-ETH10-3-186-1  2010-12-01         2        10         4   \n",
       "2277  WUR:CE-ETH10-3-186-2  2010-12-01      None      None      None   \n",
       "2278  WUR:CE-ETH10-3-186-3  2010-12-01         1         5         6   \n",
       "2279  WUR:CE-ETH10-3-186-4  2010-12-01         1        10         4   \n",
       "2280  WUR:CE-ETH10-3-187-1  2010-12-01        17        65        17   \n",
       "2281  WUR:CE-ETH10-3-187-2  2010-12-01        16        60        13   \n",
       "2282  WUR:CE-ETH10-3-187-3  2010-12-01        20        80        12   \n",
       "2283  WUR:CE-ETH10-3-187-4  2010-12-01         9        40        18   \n",
       "2284  WUR:CE-ETH10-3-188-1  2010-12-01        32       170        14   \n",
       "2285  WUR:CE-ETH10-3-188-2  2010-12-01        12        70         6   \n",
       "2286  WUR:CE-ETH10-3-188-3  2010-12-01        13       105        16   \n",
       "2287  WUR:CE-ETH10-3-188-4  2010-12-01        20       105        10   \n",
       "2288  WUR:CE-ETH10-3-189-1  2010-12-01         3        10        11   \n",
       "2289  WUR:CE-ETH10-3-189-2  2010-12-01         1         5         7   \n",
       "2290  WUR:CE-ETH10-3-189-3  2010-12-01      None      None         4   \n",
       "2291  WUR:CE-ETH10-3-189-4  2010-12-01         2        10        11   \n",
       "2292  WUR:CE-ETH10-3-190-1  2010-12-01         4        20        10   \n",
       "2293  WUR:CE-ETH10-3-190-2  2010-12-01      None      None      None   \n",
       "2294  WUR:CE-ETH10-3-190-3  2010-12-01         1         5         7   \n",
       "2295  WUR:CE-ETH10-3-190-4  2010-12-01         2        15        13   \n",
       "2296  WUR:CE-ETH10-3-191-1  2010-12-01        12        55        10   \n",
       "2297  WUR:CE-ETH10-3-191-2  2010-12-01        14        40        15   \n",
       "2298  WUR:CE-ETH10-3-191-3  2010-12-01        14       105        14   \n",
       "2299  WUR:CE-ETH10-3-191-4  2010-12-01         6        15         6   \n",
       "2300  WUR:CE-ETH10-3-192-1  2010-12-01         6        65         1   \n",
       "2301  WUR:CE-ETH10-3-192-2  2010-12-01         9        30         6   \n",
       "2302  WUR:CE-ETH10-3-192-3  2010-12-01         6        35         6   \n",
       "2303  WUR:CE-ETH10-3-192-4  2010-12-01         4        30         3   \n",
       "\n",
       "     TubW_GE20  TubN_total_per_plant  TubW_total_per_plant  \n",
       "0          410                    20                   550  \n",
       "1          145                     5                   191  \n",
       "2          460                    14                   535  \n",
       "3          185                     7                   250  \n",
       "4          255                    11                   341  \n",
       "5          705                    23                   785  \n",
       "6          340                    19                   483  \n",
       "7          280                     8                   365  \n",
       "8          305                    25                   409  \n",
       "9           95                    14                   205  \n",
       "10         400                    18                   490  \n",
       "11         270                    26                   495  \n",
       "12         425                    26                   571  \n",
       "13         526                    27                   636  \n",
       "14         505                    24                   610  \n",
       "15         420                    16                   505  \n",
       "16         175                     7                   190  \n",
       "17         260                    12                   355  \n",
       "18         155                    11                   250  \n",
       "19         105                    16                   235  \n",
       "20         560                    13                   585  \n",
       "21         555                    11                   555  \n",
       "22         140                     3                   140  \n",
       "23         110                     4                   130  \n",
       "24         185                    29                   295  \n",
       "25         560                    67                   750  \n",
       "26         415                    37                   530  \n",
       "27         450                    42                   630  \n",
       "28         520                    13                   530  \n",
       "29         395                    12                   480  \n",
       "...        ...                   ...                   ...  \n",
       "2274       210                    14                   225  \n",
       "2275       180                    12                   200  \n",
       "2276       205                     6                   215  \n",
       "2277      None                     0                     0  \n",
       "2278       185                     7                   190  \n",
       "2279       110                     5                   120  \n",
       "2280       420                    34                   485  \n",
       "2281       325                    29                   385  \n",
       "2282       195                    32                   275  \n",
       "2283       415                    27                   455  \n",
       "2284       225                    46                   395  \n",
       "2285       115                    18                   185  \n",
       "2286       300                    29                   405  \n",
       "2287       240                    30                   345  \n",
       "2288       215                    14                   225  \n",
       "2289       195                     8                   200  \n",
       "2290       105                     4                   105  \n",
       "2291       160                    13                   170  \n",
       "2292       395                    14                   415  \n",
       "2293      None                     0                     0  \n",
       "2294       320                     8                   325  \n",
       "2295       830                    15                   845  \n",
       "2296       160                    22                   215  \n",
       "2297       280                    29                   320  \n",
       "2298       195                    28                   300  \n",
       "2299        95                    12                   110  \n",
       "2300        35                     7                   100  \n",
       "2301       215                    15                   245  \n",
       "2302       125                    12                   160  \n",
       "2303       105                     7                   135  \n",
       "\n",
       "[2304 rows x 8 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2010ET =  pd.read_csv(data_or + 'data_2010ET.csv', sep='\\t')\n",
    "df_2010ET = df_2010ET.where(pd.notnull(df_2010ET), None)\n",
    "df_2010ET['Date']= pd.to_datetime(df_2010ET['Date']).dt.date\n",
    "df_2010ET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Are there any missing values?  True\n",
      "Are there any missing values in the Date column?  False\n",
      "How many missing values are there per column?\n",
      " Observation unit ID       0\n",
      "Date                      0\n",
      "TubN_LT20               298\n",
      "TubW_LT20               300\n",
      "TubN_GE20               110\n",
      "TubW_GE20               112\n",
      "TubN_total_per_plant      0\n",
      "TubW_total_per_plant      0\n",
      "dtype: int64\n",
      "How many missing values are there overall?  820\n",
      "Total cells: 18432\n",
      "Existing values: 17612\n",
      "Missing value percentage: 4.45%\n"
     ]
    }
   ],
   "source": [
    "# look at data info\n",
    "df_2010ET;\n",
    "df_2010ET.dtypes;\n",
    "# df_2003VE.info();\n",
    "# with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also\n",
    "#    print(df_1999NL)\n",
    "\n",
    "# examining missing values\n",
    "print('Are there any missing values? ', df_2010ET.isnull().values.any())\n",
    "print('Are there any missing values in the Date column? ',df_2010ET['Date'].isnull().values.any())\n",
    "print('How many missing values are there per column?\\n', df_2010ET.isnull().sum())\n",
    "missing = df_2010ET.isnull().sum().sum()\n",
    "print('How many missing values are there overall? ', missing)\n",
    "total_cells = len(df_2010ET.columns) * len(df_2010ET)\n",
    "print('Total cells: ' + str(total_cells))\n",
    "print('Existing values: ' + str(total_cells - missing))\n",
    "print('Missing value percentage: ' + str( \n",
    "      round(missing * 100 / total_cells,2)) + '%')\n",
    "\n",
    "# g.remove((None, None, None))\n",
    "g = parseStudyData('data_2010ET.csv', df_2010ET, nodes)\n",
    "#print(g_1999NL.serialize(format='ttl').decode('utf8'))\n",
    "g.serialize(destination=data_gen + 'data_2010ET.ttl', format='ttl')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "df_chuck =  pd.read_csv('etTest.csv', sep='\\t')\n",
    "df_chuck = df_chuck.where(pd.notnull(df_chuck), None)\n",
    "df_chuck['Date']= pd.to_datetime(df_chuck['Date']).dt.date\n",
    "df_chuck\n",
    "\n",
    "# g.remove((None, None, None))\n",
    "g = parseStudyData('data_2010ET.csv', df_chuck, nodes)\n",
    "#print(g_1999NL.serialize(format='ttl').decode('utf8'))\n",
    "g.serialize(destination='data-generated' + os.sep + 'data_chuck.ttl', format='nt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finland 2004 data to ttl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Observation unit ID</th>\n",
       "      <th>Date</th>\n",
       "      <th>DoE</th>\n",
       "      <th>NumMainStems</th>\n",
       "      <th>PlantHeight</th>\n",
       "      <th>TubSizeN_12-20</th>\n",
       "      <th>TubSizeN_21-30</th>\n",
       "      <th>TubSizeN_31-40</th>\n",
       "      <th>TubSizeN_41-50</th>\n",
       "      <th>TubSizeN_51-60</th>\n",
       "      <th>TubSizeN_61-70</th>\n",
       "      <th>TubSizeW_12-20</th>\n",
       "      <th>TubSizeW_21-30</th>\n",
       "      <th>TubSizeW_31-40</th>\n",
       "      <th>TubSizeW_41-50</th>\n",
       "      <th>TubSizeW_51-60</th>\n",
       "      <th>TubSizeW_61-70</th>\n",
       "      <th>TubN_total_per_plant</th>\n",
       "      <th>TubW_total_per_plant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WUR:CE-FIN04-1-1-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WUR:CE-FIN04-1-1-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WUR:CE-FIN04-1-1-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WUR:CE-FIN04-1-2-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "      <td>9.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WUR:CE-FIN04-1-2-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>WUR:CE-FIN04-1-2-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>17</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>WUR:CE-FIN04-1-3-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>8.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>WUR:CE-FIN04-1-3-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "      <td>6.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>WUR:CE-FIN04-1-3-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>WUR:CE-FIN04-1-4-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>WUR:CE-FIN04-1-4-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>7.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>WUR:CE-FIN04-1-4-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>9.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>WUR:CE-FIN04-1-5-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>9.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>WUR:CE-FIN04-1-5-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>WUR:CE-FIN04-1-5-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>WUR:CE-FIN04-1-6-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>WUR:CE-FIN04-1-6-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>WUR:CE-FIN04-1-6-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>12.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>WUR:CE-FIN04-1-7-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>14</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>WUR:CE-FIN04-1-7-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>8.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>WUR:CE-FIN04-1-7-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>WUR:CE-FIN04-1-8-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>WUR:CE-FIN04-1-8-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>WUR:CE-FIN04-1-8-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>8.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>WUR:CE-FIN04-1-9-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>WUR:CE-FIN04-1-9-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>WUR:CE-FIN04-1-9-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>WUR:CE-FIN04-1-10-A</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>18</td>\n",
       "      <td>4</td>\n",
       "      <td>10.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>WUR:CE-FIN04-1-10-B</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>11.5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>WUR:CE-FIN04-1-10-C</td>\n",
       "      <td>2004-06-30</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4599</th>\n",
       "      <td>WUR:CE-FIN04-8-1724-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>250.04</td>\n",
       "      <td>39.85</td>\n",
       "      <td>13.6</td>\n",
       "      <td>187.35</td>\n",
       "      <td>561.1</td>\n",
       "      <td>1134.85</td>\n",
       "      <td>8</td>\n",
       "      <td>2186.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4600</th>\n",
       "      <td>WUR:CE-FIN04-8-1724-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>187.19</td>\n",
       "      <td>17.49</td>\n",
       "      <td>33.85</td>\n",
       "      <td>250.21</td>\n",
       "      <td>666.58</td>\n",
       "      <td>1282.94</td>\n",
       "      <td>22</td>\n",
       "      <td>2438.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4601</th>\n",
       "      <td>WUR:CE-FIN04-8-1724-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>684.69</td>\n",
       "      <td>277.78</td>\n",
       "      <td>44.44</td>\n",
       "      <td>11.11</td>\n",
       "      <td>177.78</td>\n",
       "      <td>544.44</td>\n",
       "      <td>3</td>\n",
       "      <td>1740.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4602</th>\n",
       "      <td>WUR:CE-FIN04-8-1732-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>132.97</td>\n",
       "      <td>4.13</td>\n",
       "      <td>63.5</td>\n",
       "      <td>322.88</td>\n",
       "      <td>782.25</td>\n",
       "      <td>1441.63</td>\n",
       "      <td>16</td>\n",
       "      <td>2747.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4603</th>\n",
       "      <td>WUR:CE-FIN04-8-1732-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>581.55</td>\n",
       "      <td>213.61</td>\n",
       "      <td>21.3</td>\n",
       "      <td>28.99</td>\n",
       "      <td>236.69</td>\n",
       "      <td>644.38</td>\n",
       "      <td>13</td>\n",
       "      <td>1726.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4604</th>\n",
       "      <td>WUR:CE-FIN04-8-1732-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>23</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>314.26</td>\n",
       "      <td>67.69</td>\n",
       "      <td>3.14</td>\n",
       "      <td>138.6</td>\n",
       "      <td>474.05</td>\n",
       "      <td>1009.51</td>\n",
       "      <td>11</td>\n",
       "      <td>2007.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4605</th>\n",
       "      <td>WUR:CE-FIN04-8-1744-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>380.25</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>400</td>\n",
       "      <td>900</td>\n",
       "      <td>5</td>\n",
       "      <td>1880.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4606</th>\n",
       "      <td>WUR:CE-FIN04-8-1744-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>28</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>380.25</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>400</td>\n",
       "      <td>900</td>\n",
       "      <td>8</td>\n",
       "      <td>1880.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4607</th>\n",
       "      <td>WUR:CE-FIN04-8-1744-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>236.39</td>\n",
       "      <td>34.52</td>\n",
       "      <td>17.02</td>\n",
       "      <td>199.52</td>\n",
       "      <td>582.02</td>\n",
       "      <td>1164.52</td>\n",
       "      <td>24</td>\n",
       "      <td>2233.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4608</th>\n",
       "      <td>WUR:CE-FIN04-8-1747-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>46</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>54.55</td>\n",
       "      <td>4.47</td>\n",
       "      <td>146.75</td>\n",
       "      <td>489.03</td>\n",
       "      <td>1031.3</td>\n",
       "      <td>1773.58</td>\n",
       "      <td>79</td>\n",
       "      <td>3499.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4609</th>\n",
       "      <td>WUR:CE-FIN04-8-1747-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>19</td>\n",
       "      <td>26</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>43.99</td>\n",
       "      <td>8.22</td>\n",
       "      <td>165.57</td>\n",
       "      <td>522.92</td>\n",
       "      <td>1080.26</td>\n",
       "      <td>1837.61</td>\n",
       "      <td>49</td>\n",
       "      <td>3658.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4610</th>\n",
       "      <td>WUR:CE-FIN04-8-1747-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>21</td>\n",
       "      <td>32</td>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>56.87</td>\n",
       "      <td>3.84</td>\n",
       "      <td>143.02</td>\n",
       "      <td>482.2</td>\n",
       "      <td>1021.38</td>\n",
       "      <td>1760.56</td>\n",
       "      <td>61</td>\n",
       "      <td>3467.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4611</th>\n",
       "      <td>WUR:CE-FIN04-8-1748-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>250.04</td>\n",
       "      <td>39.85</td>\n",
       "      <td>13.6</td>\n",
       "      <td>187.35</td>\n",
       "      <td>561.1</td>\n",
       "      <td>1134.85</td>\n",
       "      <td>8</td>\n",
       "      <td>2186.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4612</th>\n",
       "      <td>WUR:CE-FIN04-8-1748-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>383.51</td>\n",
       "      <td>101.67</td>\n",
       "      <td>0.01</td>\n",
       "      <td>98.34</td>\n",
       "      <td>396.67</td>\n",
       "      <td>895.01</td>\n",
       "      <td>12</td>\n",
       "      <td>1875.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4613</th>\n",
       "      <td>WUR:CE-FIN04-8-1748-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>201.42</td>\n",
       "      <td>22.02</td>\n",
       "      <td>28.17</td>\n",
       "      <td>234.33</td>\n",
       "      <td>640.48</td>\n",
       "      <td>1246.63</td>\n",
       "      <td>13</td>\n",
       "      <td>2373.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4614</th>\n",
       "      <td>WUR:CE-FIN04-8-1755-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>870.25</td>\n",
       "      <td>400</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>400</td>\n",
       "      <td>16</td>\n",
       "      <td>1870.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4615</th>\n",
       "      <td>WUR:CE-FIN04-8-1755-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>213.16</td>\n",
       "      <td>26.01</td>\n",
       "      <td>24.01</td>\n",
       "      <td>222.01</td>\n",
       "      <td>620.01</td>\n",
       "      <td>1218.01</td>\n",
       "      <td>10</td>\n",
       "      <td>2323.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4616</th>\n",
       "      <td>WUR:CE-FIN04-8-1755-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>16</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>751.91</td>\n",
       "      <td>321.16</td>\n",
       "      <td>62.74</td>\n",
       "      <td>4.32</td>\n",
       "      <td>145.9</td>\n",
       "      <td>487.48</td>\n",
       "      <td>19</td>\n",
       "      <td>1773.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4617</th>\n",
       "      <td>WUR:CE-FIN04-8-1758-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>15</td>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>215.11</td>\n",
       "      <td>26.69</td>\n",
       "      <td>23.36</td>\n",
       "      <td>220.03</td>\n",
       "      <td>616.69</td>\n",
       "      <td>1213.36</td>\n",
       "      <td>30</td>\n",
       "      <td>2315.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4618</th>\n",
       "      <td>WUR:CE-FIN04-8-1758-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>177.78</td>\n",
       "      <td>14.69</td>\n",
       "      <td>38.03</td>\n",
       "      <td>261.36</td>\n",
       "      <td>684.69</td>\n",
       "      <td>1308.03</td>\n",
       "      <td>24</td>\n",
       "      <td>2484.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4619</th>\n",
       "      <td>WUR:CE-FIN04-8-1758-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>260.89</td>\n",
       "      <td>44.25</td>\n",
       "      <td>11.21</td>\n",
       "      <td>178.16</td>\n",
       "      <td>545.12</td>\n",
       "      <td>1112.08</td>\n",
       "      <td>23</td>\n",
       "      <td>2151.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4620</th>\n",
       "      <td>WUR:CE-FIN04-8-1762-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>456.4</td>\n",
       "      <td>140.75</td>\n",
       "      <td>3.47</td>\n",
       "      <td>66.2</td>\n",
       "      <td>328.93</td>\n",
       "      <td>791.65</td>\n",
       "      <td>22</td>\n",
       "      <td>1787.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4621</th>\n",
       "      <td>WUR:CE-FIN04-8-1762-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>484</td>\n",
       "      <td>156.25</td>\n",
       "      <td>6.25</td>\n",
       "      <td>56.25</td>\n",
       "      <td>306.25</td>\n",
       "      <td>756.25</td>\n",
       "      <td>8</td>\n",
       "      <td>1765.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4622</th>\n",
       "      <td>WUR:CE-FIN04-8-1762-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>477.25</td>\n",
       "      <td>152.43</td>\n",
       "      <td>5.5</td>\n",
       "      <td>58.58</td>\n",
       "      <td>311.66</td>\n",
       "      <td>764.74</td>\n",
       "      <td>13</td>\n",
       "      <td>1770.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4623</th>\n",
       "      <td>WUR:CE-FIN04-8-1765-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>105.06</td>\n",
       "      <td>0.56</td>\n",
       "      <td>85.56</td>\n",
       "      <td>370.56</td>\n",
       "      <td>855.56</td>\n",
       "      <td>1540.56</td>\n",
       "      <td>36</td>\n",
       "      <td>2957.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4624</th>\n",
       "      <td>WUR:CE-FIN04-8-1765-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>14</td>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>162.38</td>\n",
       "      <td>10.52</td>\n",
       "      <td>45.66</td>\n",
       "      <td>280.8</td>\n",
       "      <td>715.94</td>\n",
       "      <td>1351.09</td>\n",
       "      <td>35</td>\n",
       "      <td>2566.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4625</th>\n",
       "      <td>WUR:CE-FIN04-8-1765-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>248.06</td>\n",
       "      <td>39.06</td>\n",
       "      <td>14.06</td>\n",
       "      <td>189.06</td>\n",
       "      <td>564.06</td>\n",
       "      <td>1139.06</td>\n",
       "      <td>26</td>\n",
       "      <td>2193.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4626</th>\n",
       "      <td>WUR:CE-FIN04-8-1769-A</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>342.25</td>\n",
       "      <td>81</td>\n",
       "      <td>1</td>\n",
       "      <td>121</td>\n",
       "      <td>441</td>\n",
       "      <td>961</td>\n",
       "      <td>18</td>\n",
       "      <td>1947.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4627</th>\n",
       "      <td>WUR:CE-FIN04-8-1769-B</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>345.18</td>\n",
       "      <td>82.43</td>\n",
       "      <td>0.85</td>\n",
       "      <td>119.27</td>\n",
       "      <td>437.69</td>\n",
       "      <td>956.11</td>\n",
       "      <td>19</td>\n",
       "      <td>1941.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4628</th>\n",
       "      <td>WUR:CE-FIN04-8-1769-C</td>\n",
       "      <td>2004-10-07</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>233.65</td>\n",
       "      <td>33.47</td>\n",
       "      <td>17.76</td>\n",
       "      <td>202.05</td>\n",
       "      <td>586.33</td>\n",
       "      <td>1170.62</td>\n",
       "      <td>28</td>\n",
       "      <td>2243.88</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4629 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Observation unit ID        Date DoE NumMainStems PlantHeight  \\\n",
       "0        WUR:CE-FIN04-1-1-A  2004-06-30  16            3           7   \n",
       "1        WUR:CE-FIN04-1-1-B  2004-06-30  19            4           9   \n",
       "2        WUR:CE-FIN04-1-1-C  2004-06-30  19            2           6   \n",
       "3        WUR:CE-FIN04-1-2-A  2004-06-30  18            3         9.5   \n",
       "4        WUR:CE-FIN04-1-2-B  2004-06-30  18            3          11   \n",
       "5        WUR:CE-FIN04-1-2-C  2004-06-30  17            5          12   \n",
       "6        WUR:CE-FIN04-1-3-A  2004-06-30  14            4         8.5   \n",
       "7        WUR:CE-FIN04-1-3-B  2004-06-30  15            4         6.5   \n",
       "8        WUR:CE-FIN04-1-3-C  2004-06-30  15            4           6   \n",
       "9        WUR:CE-FIN04-1-4-A  2004-06-30  27            1           1   \n",
       "10       WUR:CE-FIN04-1-4-B  2004-06-30  18            2         7.5   \n",
       "11       WUR:CE-FIN04-1-4-C  2004-06-30  17            6         9.5   \n",
       "12       WUR:CE-FIN04-1-5-A  2004-06-30  15            2         9.5   \n",
       "13       WUR:CE-FIN04-1-5-B  2004-06-30  15            2          11   \n",
       "14       WUR:CE-FIN04-1-5-C  2004-06-30  16            2          12   \n",
       "15       WUR:CE-FIN04-1-6-A  2004-06-30  17            1          10   \n",
       "16       WUR:CE-FIN04-1-6-B  2004-06-30  16            2          10   \n",
       "17       WUR:CE-FIN04-1-6-C  2004-06-30  17            1        12.5   \n",
       "18       WUR:CE-FIN04-1-7-A  2004-06-30  14            6          12   \n",
       "19       WUR:CE-FIN04-1-7-B  2004-06-30  14            3         8.5   \n",
       "20       WUR:CE-FIN04-1-7-C  2004-06-30  14            2          11   \n",
       "21       WUR:CE-FIN04-1-8-A  2004-06-30  17            2           9   \n",
       "22       WUR:CE-FIN04-1-8-B  2004-06-30  17            1          10   \n",
       "23       WUR:CE-FIN04-1-8-C  2004-06-30  17            2         8.5   \n",
       "24       WUR:CE-FIN04-1-9-A  2004-06-30  15            4           7   \n",
       "25       WUR:CE-FIN04-1-9-B  2004-06-30  15            4           7   \n",
       "26       WUR:CE-FIN04-1-9-C  2004-06-30  21            3           6   \n",
       "27      WUR:CE-FIN04-1-10-A  2004-06-30  18            4        10.5   \n",
       "28      WUR:CE-FIN04-1-10-B  2004-06-30  17            4        11.5   \n",
       "29      WUR:CE-FIN04-1-10-C  2004-06-30  17            1          10   \n",
       "...                     ...         ...  ..          ...         ...   \n",
       "4599  WUR:CE-FIN04-8-1724-A  2004-10-07  18            1        None   \n",
       "4600  WUR:CE-FIN04-8-1724-B  2004-10-07  17            2        None   \n",
       "4601  WUR:CE-FIN04-8-1724-C  2004-10-07  23            1        None   \n",
       "4602  WUR:CE-FIN04-8-1732-A  2004-10-07  17            2        None   \n",
       "4603  WUR:CE-FIN04-8-1732-B  2004-10-07  15            3        None   \n",
       "4604  WUR:CE-FIN04-8-1732-C  2004-10-07  23            2        None   \n",
       "4605  WUR:CE-FIN04-8-1744-A  2004-10-07  19            2        None   \n",
       "4606  WUR:CE-FIN04-8-1744-B  2004-10-07  28            2        None   \n",
       "4607  WUR:CE-FIN04-8-1744-C  2004-10-07  20            6        None   \n",
       "4608  WUR:CE-FIN04-8-1747-A  2004-10-07  18            2        None   \n",
       "4609  WUR:CE-FIN04-8-1747-B  2004-10-07  15            1        None   \n",
       "4610  WUR:CE-FIN04-8-1747-C  2004-10-07  18            2        None   \n",
       "4611  WUR:CE-FIN04-8-1748-A  2004-10-07  18            2        None   \n",
       "4612  WUR:CE-FIN04-8-1748-B  2004-10-07  17            1        None   \n",
       "4613  WUR:CE-FIN04-8-1748-C  2004-10-07  17            3        None   \n",
       "4614  WUR:CE-FIN04-8-1755-A  2004-10-07  17            3        None   \n",
       "4615  WUR:CE-FIN04-8-1755-B  2004-10-07  18            2        None   \n",
       "4616  WUR:CE-FIN04-8-1755-C  2004-10-07  16            4        None   \n",
       "4617  WUR:CE-FIN04-8-1758-A  2004-10-07  15            7        None   \n",
       "4618  WUR:CE-FIN04-8-1758-B  2004-10-07  15            4        None   \n",
       "4619  WUR:CE-FIN04-8-1758-C  2004-10-07  18            5        None   \n",
       "4620  WUR:CE-FIN04-8-1762-A  2004-10-07  18            5        None   \n",
       "4621  WUR:CE-FIN04-8-1762-B  2004-10-07  19            2        None   \n",
       "4622  WUR:CE-FIN04-8-1762-C  2004-10-07  20            4        None   \n",
       "4623  WUR:CE-FIN04-8-1765-A  2004-10-07  14            4        None   \n",
       "4624  WUR:CE-FIN04-8-1765-B  2004-10-07  14            8        None   \n",
       "4625  WUR:CE-FIN04-8-1765-C  2004-10-07  14            5        None   \n",
       "4626  WUR:CE-FIN04-8-1769-A  2004-10-07  15            6        None   \n",
       "4627  WUR:CE-FIN04-8-1769-B  2004-10-07  15            5        None   \n",
       "4628  WUR:CE-FIN04-8-1769-C  2004-10-07  15            6        None   \n",
       "\n",
       "     TubSizeN_12-20 TubSizeN_21-30 TubSizeN_31-40 TubSizeN_41-50  \\\n",
       "0              None           None           None           None   \n",
       "1              None           None           None           None   \n",
       "2              None           None           None           None   \n",
       "3              None           None           None           None   \n",
       "4              None           None           None           None   \n",
       "5              None           None           None           None   \n",
       "6              None           None           None           None   \n",
       "7              None           None           None           None   \n",
       "8              None           None           None           None   \n",
       "9              None           None           None           None   \n",
       "10             None           None           None           None   \n",
       "11             None           None           None           None   \n",
       "12             None           None           None           None   \n",
       "13             None           None           None           None   \n",
       "14             None           None           None           None   \n",
       "15             None           None           None           None   \n",
       "16             None           None           None           None   \n",
       "17             None           None           None           None   \n",
       "18             None           None           None           None   \n",
       "19             None           None           None           None   \n",
       "20             None           None           None           None   \n",
       "21             None           None           None           None   \n",
       "22             None           None           None           None   \n",
       "23             None           None           None           None   \n",
       "24             None           None           None           None   \n",
       "25             None           None           None           None   \n",
       "26             None           None           None           None   \n",
       "27             None           None           None           None   \n",
       "28             None           None           None           None   \n",
       "29             None           None           None           None   \n",
       "...             ...            ...            ...            ...   \n",
       "4599              1              3              2              2   \n",
       "4600              4              7              9              2   \n",
       "4601           None           None              1              2   \n",
       "4602              5              6              3              1   \n",
       "4603           None              2              5              4   \n",
       "4604              1              3              4              3   \n",
       "4605           None              2              2           None   \n",
       "4606           None              3              2              3   \n",
       "4607              2             11              6              5   \n",
       "4608             26             46              6              1   \n",
       "4609             19             26              4           None   \n",
       "4610             21             32              8           None   \n",
       "4611              1              3              2              2   \n",
       "4612              2              3              2              3   \n",
       "4613              2              4              6              1   \n",
       "4614           None              1              4              5   \n",
       "4615              2              3              3              2   \n",
       "4616              1           None              5              9   \n",
       "4617             10              2             12              5   \n",
       "4618              4              8             11              1   \n",
       "4619              6              3              7              7   \n",
       "4620              2              4              8              4   \n",
       "4621           None              1              5              1   \n",
       "4622              1              1              5              6   \n",
       "4623             14             13              4              3   \n",
       "4624              7             15             11           None   \n",
       "4625              5              5             13              1   \n",
       "4626              4              2              6              4   \n",
       "4627              5              2              5              4   \n",
       "4628              4             10              8              6   \n",
       "\n",
       "     TubSizeN_51-60 TubSizeN_61-70 TubSizeW_12-20 TubSizeW_21-30  \\\n",
       "0              None           None           None           None   \n",
       "1              None           None           None           None   \n",
       "2              None           None           None           None   \n",
       "3              None           None           None           None   \n",
       "4              None           None           None           None   \n",
       "5              None           None           None           None   \n",
       "6              None           None           None           None   \n",
       "7              None           None           None           None   \n",
       "8              None           None           None           None   \n",
       "9              None           None           None           None   \n",
       "10             None           None           None           None   \n",
       "11             None           None           None           None   \n",
       "12             None           None           None           None   \n",
       "13             None           None           None           None   \n",
       "14             None           None           None           None   \n",
       "15             None           None           None           None   \n",
       "16             None           None           None           None   \n",
       "17             None           None           None           None   \n",
       "18             None           None           None           None   \n",
       "19             None           None           None           None   \n",
       "20             None           None           None           None   \n",
       "21             None           None           None           None   \n",
       "22             None           None           None           None   \n",
       "23             None           None           None           None   \n",
       "24             None           None           None           None   \n",
       "25             None           None           None           None   \n",
       "26             None           None           None           None   \n",
       "27             None           None           None           None   \n",
       "28             None           None           None           None   \n",
       "29             None           None           None           None   \n",
       "...             ...            ...            ...            ...   \n",
       "4599           None           None         250.04          39.85   \n",
       "4600           None           None         187.19          17.49   \n",
       "4601           None           None         684.69         277.78   \n",
       "4602              1           None         132.97           4.13   \n",
       "4603              2           None         581.55         213.61   \n",
       "4604           None           None         314.26          67.69   \n",
       "4605              1           None         380.25            100   \n",
       "4606           None           None         380.25            100   \n",
       "4607           None           None         236.39          34.52   \n",
       "4608           None           None          54.55           4.47   \n",
       "4609           None           None          43.99           8.22   \n",
       "4610           None           None          56.87           3.84   \n",
       "4611           None           None         250.04          39.85   \n",
       "4612              2           None         383.51         101.67   \n",
       "4613           None           None         201.42          22.02   \n",
       "4614              6           None         870.25            400   \n",
       "4615           None           None         213.16          26.01   \n",
       "4616              4           None         751.91         321.16   \n",
       "4617              1           None         215.11          26.69   \n",
       "4618           None           None         177.78          14.69   \n",
       "4619           None           None         260.89          44.25   \n",
       "4620              4           None          456.4         140.75   \n",
       "4621              1           None            484         156.25   \n",
       "4622           None           None         477.25         152.43   \n",
       "4623              2           None         105.06           0.56   \n",
       "4624              1              1         162.38          10.52   \n",
       "4625              2           None         248.06          39.06   \n",
       "4626              2           None         342.25             81   \n",
       "4627              3           None         345.18          82.43   \n",
       "4628           None           None         233.65          33.47   \n",
       "\n",
       "     TubSizeW_31-40 TubSizeW_41-50 TubSizeW_51-60 TubSizeW_61-70  \\\n",
       "0              None           None           None           None   \n",
       "1              None           None           None           None   \n",
       "2              None           None           None           None   \n",
       "3              None           None           None           None   \n",
       "4              None           None           None           None   \n",
       "5              None           None           None           None   \n",
       "6              None           None           None           None   \n",
       "7              None           None           None           None   \n",
       "8              None           None           None           None   \n",
       "9              None           None           None           None   \n",
       "10             None           None           None           None   \n",
       "11             None           None           None           None   \n",
       "12             None           None           None           None   \n",
       "13             None           None           None           None   \n",
       "14             None           None           None           None   \n",
       "15             None           None           None           None   \n",
       "16             None           None           None           None   \n",
       "17             None           None           None           None   \n",
       "18             None           None           None           None   \n",
       "19             None           None           None           None   \n",
       "20             None           None           None           None   \n",
       "21             None           None           None           None   \n",
       "22             None           None           None           None   \n",
       "23             None           None           None           None   \n",
       "24             None           None           None           None   \n",
       "25             None           None           None           None   \n",
       "26             None           None           None           None   \n",
       "27             None           None           None           None   \n",
       "28             None           None           None           None   \n",
       "29             None           None           None           None   \n",
       "...             ...            ...            ...            ...   \n",
       "4599           13.6         187.35          561.1        1134.85   \n",
       "4600          33.85         250.21         666.58        1282.94   \n",
       "4601          44.44          11.11         177.78         544.44   \n",
       "4602           63.5         322.88         782.25        1441.63   \n",
       "4603           21.3          28.99         236.69         644.38   \n",
       "4604           3.14          138.6         474.05        1009.51   \n",
       "4605              0            100            400            900   \n",
       "4606              0            100            400            900   \n",
       "4607          17.02         199.52         582.02        1164.52   \n",
       "4608         146.75         489.03         1031.3        1773.58   \n",
       "4609         165.57         522.92        1080.26        1837.61   \n",
       "4610         143.02          482.2        1021.38        1760.56   \n",
       "4611           13.6         187.35          561.1        1134.85   \n",
       "4612           0.01          98.34         396.67         895.01   \n",
       "4613          28.17         234.33         640.48        1246.63   \n",
       "4614            100              0            100            400   \n",
       "4615          24.01         222.01         620.01        1218.01   \n",
       "4616          62.74           4.32          145.9         487.48   \n",
       "4617          23.36         220.03         616.69        1213.36   \n",
       "4618          38.03         261.36         684.69        1308.03   \n",
       "4619          11.21         178.16         545.12        1112.08   \n",
       "4620           3.47           66.2         328.93         791.65   \n",
       "4621           6.25          56.25         306.25         756.25   \n",
       "4622            5.5          58.58         311.66         764.74   \n",
       "4623          85.56         370.56         855.56        1540.56   \n",
       "4624          45.66          280.8         715.94        1351.09   \n",
       "4625          14.06         189.06         564.06        1139.06   \n",
       "4626              1            121            441            961   \n",
       "4627           0.85         119.27         437.69         956.11   \n",
       "4628          17.76         202.05         586.33        1170.62   \n",
       "\n",
       "     TubN_total_per_plant TubW_total_per_plant  \n",
       "0                    None                 None  \n",
       "1                    None                 None  \n",
       "2                    None                 None  \n",
       "3                    None                 None  \n",
       "4                    None                 None  \n",
       "5                    None                 None  \n",
       "6                    None                 None  \n",
       "7                    None                 None  \n",
       "8                    None                 None  \n",
       "9                    None                 None  \n",
       "10                   None                 None  \n",
       "11                   None                 None  \n",
       "12                   None                 None  \n",
       "13                   None                 None  \n",
       "14                   None                 None  \n",
       "15                   None                 None  \n",
       "16                   None                 None  \n",
       "17                   None                 None  \n",
       "18                   None                 None  \n",
       "19                   None                 None  \n",
       "20                   None                 None  \n",
       "21                   None                 None  \n",
       "22                   None                 None  \n",
       "23                   None                 None  \n",
       "24                   None                 None  \n",
       "25                   None                 None  \n",
       "26                   None                 None  \n",
       "27                   None                 None  \n",
       "28                   None                 None  \n",
       "29                   None                 None  \n",
       "...                   ...                  ...  \n",
       "4599                    8              2186.77  \n",
       "4600                   22              2438.27  \n",
       "4601                    3              1740.25  \n",
       "4602                   16              2747.35  \n",
       "4603                   13              1726.52  \n",
       "4604                   11              2007.24  \n",
       "4605                    5              1880.25  \n",
       "4606                    8              1880.25  \n",
       "4607                   24              2233.97  \n",
       "4608                   79              3499.68  \n",
       "4609                   49              3658.57  \n",
       "4610                   61              3467.86  \n",
       "4611                    8              2186.77  \n",
       "4612                   12              1875.21  \n",
       "4613                   13              2373.05  \n",
       "4614                   16              1870.25  \n",
       "4615                   10              2323.21  \n",
       "4616                   19              1773.52  \n",
       "4617                   30              2315.25  \n",
       "4618                   24              2484.58  \n",
       "4619                   23              2151.72  \n",
       "4620                   22              1787.41  \n",
       "4621                    8              1765.25  \n",
       "4622                   13              1770.16  \n",
       "4623                   36              2957.88  \n",
       "4624                   35              2566.39  \n",
       "4625                   26              2193.38  \n",
       "4626                   18              1947.25  \n",
       "4627                   19              1941.52  \n",
       "4628                   28              2243.88  \n",
       "\n",
       "[4629 rows x 19 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2004Fin =  pd.read_csv(data_or + 'data_2004Fin.csv', sep='\\t')\n",
    "df_2004Fin = df_2004Fin.where(pd.notnull(df_2004Fin), None)\n",
    "df_2004Fin['Date']= pd.to_datetime(df_2004Fin['Date']).dt.date\n",
    "df_2004Fin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4629 entries, 0 to 4628\n",
      "Data columns (total 19 columns):\n",
      "Observation unit ID     4629 non-null object\n",
      "Date                    4629 non-null object\n",
      "DoE                     4541 non-null object\n",
      "NumMainStems            4542 non-null object\n",
      "PlantHeight             3902 non-null object\n",
      "TubSizeN_12-20          565 non-null object\n",
      "TubSizeN_21-30          627 non-null object\n",
      "TubSizeN_31-40          623 non-null object\n",
      "TubSizeN_41-50          513 non-null object\n",
      "TubSizeN_51-60          199 non-null object\n",
      "TubSizeN_61-70          25 non-null object\n",
      "TubSizeW_12-20          651 non-null object\n",
      "TubSizeW_21-30          651 non-null object\n",
      "TubSizeW_31-40          651 non-null object\n",
      "TubSizeW_41-50          651 non-null object\n",
      "TubSizeW_51-60          651 non-null object\n",
      "TubSizeW_61-70          651 non-null object\n",
      "TubN_total_per_plant    651 non-null object\n",
      "TubW_total_per_plant    651 non-null object\n",
      "dtypes: object(19)\n",
      "memory usage: 343.6+ KB\n",
      "Are there any missing values?  True\n",
      "Are there any missing values in the Date column?  False\n",
      "How many missing values are there per column?\n",
      " Observation unit ID        0\n",
      "Date                       0\n",
      "DoE                       88\n",
      "NumMainStems              87\n",
      "PlantHeight              727\n",
      "TubSizeN_12-20          4064\n",
      "TubSizeN_21-30          4002\n",
      "TubSizeN_31-40          4006\n",
      "TubSizeN_41-50          4116\n",
      "TubSizeN_51-60          4430\n",
      "TubSizeN_61-70          4604\n",
      "TubSizeW_12-20          3978\n",
      "TubSizeW_21-30          3978\n",
      "TubSizeW_31-40          3978\n",
      "TubSizeW_41-50          3978\n",
      "TubSizeW_51-60          3978\n",
      "TubSizeW_61-70          3978\n",
      "TubN_total_per_plant    3978\n",
      "TubW_total_per_plant    3978\n",
      "dtype: int64\n",
      "How many missing values are there overall?  57948\n",
      "Total cells: 87951\n",
      "Existing values: 30003\n",
      "Missing value percentage: 65.89%\n"
     ]
    }
   ],
   "source": [
    "# look at data info\n",
    "df_2004Fin;\n",
    "df_2004Fin.dtypes;\n",
    "df_2004Fin.info();\n",
    "# with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also\n",
    "#    print(df_2004FinNL)\n",
    "\n",
    "# examining missing values\n",
    "print('Are there any missing values? ', df_2004Fin.isnull().values.any())\n",
    "print('Are there any missing values in the Date column? ',df_2004Fin['Date'].isnull().values.any())\n",
    "print('How many missing values are there per column?\\n', df_2004Fin.isnull().sum())\n",
    "missing = df_2004Fin.isnull().sum().sum()\n",
    "print('How many missing values are there overall? ', missing)\n",
    "total_cells = len(df_2004Fin.columns) * len(df_2004Fin)\n",
    "print('Total cells: ' + str(total_cells))\n",
    "print('Existing values: ' + str(total_cells - missing))\n",
    "print('Missing value percentage: ' + str( \n",
    "      round(missing * 100 / total_cells,2)) + '%')\n",
    "\n",
    "# g.remove((None, None, None))\n",
    "g = parseStudyData('data_2004Fin.csv', df_2004Fin, nodes)\n",
    "#print(g_2004Fin.serialize(format='ttl').decode('utf8'))\n",
    "g.serialize(destination=data_gen + 'data_2004Fin.ttl', format='ttl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finland 2005 data to ttl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Observation unit ID</th>\n",
       "      <th>Date</th>\n",
       "      <th>NumMainStems</th>\n",
       "      <th>TubN_total_per_plant</th>\n",
       "      <th>TubW_total_per_plant</th>\n",
       "      <th>DoE</th>\n",
       "      <th>PlantHeight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WUR:CE-FIN05-I-1-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>32</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WUR:CE-FIN05-I-1-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>27</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WUR:CE-FIN05-I-1-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>35</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WUR:CE-FIN05-I-2-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>25</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WUR:CE-FIN05-I-2-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>WUR:CE-FIN05-I-2-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>WUR:CE-FIN05-I-3-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>WUR:CE-FIN05-I-3-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>WUR:CE-FIN05-I-3-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>WUR:CE-FIN05-I-4-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>24</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>WUR:CE-FIN05-I-4-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>24</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>WUR:CE-FIN05-I-4-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>WUR:CE-FIN05-I-5-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>WUR:CE-FIN05-I-5-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>WUR:CE-FIN05-I-5-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>30</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>WUR:CE-FIN05-I-6-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>WUR:CE-FIN05-I-6-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>33</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>WUR:CE-FIN05-I-6-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>34</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>WUR:CE-FIN05-I-7-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>WUR:CE-FIN05-I-7-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>WUR:CE-FIN05-I-7-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>WUR:CE-FIN05-I-8-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>WUR:CE-FIN05-I-8-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>WUR:CE-FIN05-I-8-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>WUR:CE-FIN05-I-9-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>WUR:CE-FIN05-I-9-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>28</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>WUR:CE-FIN05-I-9-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>25</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>WUR:CE-FIN05-I-10-A</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>WUR:CE-FIN05-I-10-B</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>25</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>WUR:CE-FIN05-I-10-C</td>\n",
       "      <td>2005-06-28</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>28</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3057</th>\n",
       "      <td>WUR:CE-FIN05-IV-1031-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>58</td>\n",
       "      <td>2667.5</td>\n",
       "      <td>18</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3058</th>\n",
       "      <td>WUR:CE-FIN05-IV-1031-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>38</td>\n",
       "      <td>1148</td>\n",
       "      <td>18</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3059</th>\n",
       "      <td>WUR:CE-FIN05-IV-1031-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>18</td>\n",
       "      <td>1096</td>\n",
       "      <td>18</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3060</th>\n",
       "      <td>WUR:CE-FIN05-IV-1032-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>1898</td>\n",
       "      <td>18</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3061</th>\n",
       "      <td>WUR:CE-FIN05-IV-1032-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>18</td>\n",
       "      <td>1808</td>\n",
       "      <td>18</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3062</th>\n",
       "      <td>WUR:CE-FIN05-IV-1032-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>20</td>\n",
       "      <td>1345</td>\n",
       "      <td>21</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3063</th>\n",
       "      <td>WUR:CE-FIN05-IV-1033-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>25</td>\n",
       "      <td>1361.5</td>\n",
       "      <td>23</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3064</th>\n",
       "      <td>WUR:CE-FIN05-IV-1033-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>13</td>\n",
       "      <td>797</td>\n",
       "      <td>22</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3065</th>\n",
       "      <td>WUR:CE-FIN05-IV-1033-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>9</td>\n",
       "      <td>1287.5</td>\n",
       "      <td>21</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3066</th>\n",
       "      <td>WUR:CE-FIN05-IV-1034-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>21</td>\n",
       "      <td>1771.5</td>\n",
       "      <td>18</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3067</th>\n",
       "      <td>WUR:CE-FIN05-IV-1034-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>18</td>\n",
       "      <td>1674</td>\n",
       "      <td>18</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3068</th>\n",
       "      <td>WUR:CE-FIN05-IV-1034-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>45</td>\n",
       "      <td>2056</td>\n",
       "      <td>18</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3069</th>\n",
       "      <td>WUR:CE-FIN05-IV-1035-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>13</td>\n",
       "      <td>1009.5</td>\n",
       "      <td>20</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3070</th>\n",
       "      <td>WUR:CE-FIN05-IV-1035-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>10</td>\n",
       "      <td>605.5</td>\n",
       "      <td>22</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3071</th>\n",
       "      <td>WUR:CE-FIN05-IV-1035-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>11</td>\n",
       "      <td>1191.5</td>\n",
       "      <td>21</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3072</th>\n",
       "      <td>WUR:CE-FIN05-IV-1036-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>22</td>\n",
       "      <td>1004</td>\n",
       "      <td>26</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3073</th>\n",
       "      <td>WUR:CE-FIN05-IV-1036-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>13</td>\n",
       "      <td>1060</td>\n",
       "      <td>25</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3074</th>\n",
       "      <td>WUR:CE-FIN05-IV-1036-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>1002</td>\n",
       "      <td>27</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3075</th>\n",
       "      <td>WUR:CE-FIN05-IV-1037-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>525</td>\n",
       "      <td>35</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3076</th>\n",
       "      <td>WUR:CE-FIN05-IV-1037-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>12</td>\n",
       "      <td>612</td>\n",
       "      <td>35</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3077</th>\n",
       "      <td>WUR:CE-FIN05-IV-1037-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>8</td>\n",
       "      <td>428.5</td>\n",
       "      <td>35</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3078</th>\n",
       "      <td>WUR:CE-FIN05-IV-1038-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>18</td>\n",
       "      <td>577.5</td>\n",
       "      <td>32</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3079</th>\n",
       "      <td>WUR:CE-FIN05-IV-1038-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>7</td>\n",
       "      <td>524.5</td>\n",
       "      <td>28</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3080</th>\n",
       "      <td>WUR:CE-FIN05-IV-1038-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>11</td>\n",
       "      <td>545</td>\n",
       "      <td>34</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3081</th>\n",
       "      <td>WUR:CE-FIN05-IV-1039-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>202.5</td>\n",
       "      <td>36</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3082</th>\n",
       "      <td>WUR:CE-FIN05-IV-1039-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>13</td>\n",
       "      <td>657</td>\n",
       "      <td>30</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3083</th>\n",
       "      <td>WUR:CE-FIN05-IV-1039-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>8</td>\n",
       "      <td>445</td>\n",
       "      <td>33</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3084</th>\n",
       "      <td>WUR:CE-FIN05-IV-1040-A</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>16</td>\n",
       "      <td>643.5</td>\n",
       "      <td>33</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3085</th>\n",
       "      <td>WUR:CE-FIN05-IV-1040-B</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>19</td>\n",
       "      <td>688.5</td>\n",
       "      <td>31</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3086</th>\n",
       "      <td>WUR:CE-FIN05-IV-1040-C</td>\n",
       "      <td>2005-09-20</td>\n",
       "      <td>None</td>\n",
       "      <td>22</td>\n",
       "      <td>964</td>\n",
       "      <td>33</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3087 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Observation unit ID        Date NumMainStems TubN_total_per_plant  \\\n",
       "0         WUR:CE-FIN05-I-1-A  2005-06-28         None                 None   \n",
       "1         WUR:CE-FIN05-I-1-B  2005-06-28         None                 None   \n",
       "2         WUR:CE-FIN05-I-1-C  2005-06-28         None                 None   \n",
       "3         WUR:CE-FIN05-I-2-A  2005-06-28         None                 None   \n",
       "4         WUR:CE-FIN05-I-2-B  2005-06-28         None                 None   \n",
       "5         WUR:CE-FIN05-I-2-C  2005-06-28         None                 None   \n",
       "6         WUR:CE-FIN05-I-3-A  2005-06-28         None                 None   \n",
       "7         WUR:CE-FIN05-I-3-B  2005-06-28         None                 None   \n",
       "8         WUR:CE-FIN05-I-3-C  2005-06-28         None                 None   \n",
       "9         WUR:CE-FIN05-I-4-A  2005-06-28         None                 None   \n",
       "10        WUR:CE-FIN05-I-4-B  2005-06-28         None                 None   \n",
       "11        WUR:CE-FIN05-I-4-C  2005-06-28         None                 None   \n",
       "12        WUR:CE-FIN05-I-5-A  2005-06-28         None                 None   \n",
       "13        WUR:CE-FIN05-I-5-B  2005-06-28         None                 None   \n",
       "14        WUR:CE-FIN05-I-5-C  2005-06-28         None                 None   \n",
       "15        WUR:CE-FIN05-I-6-A  2005-06-28         None                 None   \n",
       "16        WUR:CE-FIN05-I-6-B  2005-06-28         None                 None   \n",
       "17        WUR:CE-FIN05-I-6-C  2005-06-28         None                 None   \n",
       "18        WUR:CE-FIN05-I-7-A  2005-06-28         None                 None   \n",
       "19        WUR:CE-FIN05-I-7-B  2005-06-28         None                 None   \n",
       "20        WUR:CE-FIN05-I-7-C  2005-06-28         None                 None   \n",
       "21        WUR:CE-FIN05-I-8-A  2005-06-28         None                 None   \n",
       "22        WUR:CE-FIN05-I-8-B  2005-06-28         None                 None   \n",
       "23        WUR:CE-FIN05-I-8-C  2005-06-28         None                 None   \n",
       "24        WUR:CE-FIN05-I-9-A  2005-06-28         None                    2   \n",
       "25        WUR:CE-FIN05-I-9-B  2005-06-28         None                 None   \n",
       "26        WUR:CE-FIN05-I-9-C  2005-06-28         None                 None   \n",
       "27       WUR:CE-FIN05-I-10-A  2005-06-28         None                 None   \n",
       "28       WUR:CE-FIN05-I-10-B  2005-06-28         None                 None   \n",
       "29       WUR:CE-FIN05-I-10-C  2005-06-28         None                 None   \n",
       "...                      ...         ...          ...                  ...   \n",
       "3057  WUR:CE-FIN05-IV-1031-A  2005-09-20         None                   58   \n",
       "3058  WUR:CE-FIN05-IV-1031-B  2005-09-20         None                   38   \n",
       "3059  WUR:CE-FIN05-IV-1031-C  2005-09-20         None                   18   \n",
       "3060  WUR:CE-FIN05-IV-1032-A  2005-09-20         None                   26   \n",
       "3061  WUR:CE-FIN05-IV-1032-B  2005-09-20         None                   18   \n",
       "3062  WUR:CE-FIN05-IV-1032-C  2005-09-20         None                   20   \n",
       "3063  WUR:CE-FIN05-IV-1033-A  2005-09-20         None                   25   \n",
       "3064  WUR:CE-FIN05-IV-1033-B  2005-09-20         None                   13   \n",
       "3065  WUR:CE-FIN05-IV-1033-C  2005-09-20         None                    9   \n",
       "3066  WUR:CE-FIN05-IV-1034-A  2005-09-20         None                   21   \n",
       "3067  WUR:CE-FIN05-IV-1034-B  2005-09-20         None                   18   \n",
       "3068  WUR:CE-FIN05-IV-1034-C  2005-09-20         None                   45   \n",
       "3069  WUR:CE-FIN05-IV-1035-A  2005-09-20         None                   13   \n",
       "3070  WUR:CE-FIN05-IV-1035-B  2005-09-20         None                   10   \n",
       "3071  WUR:CE-FIN05-IV-1035-C  2005-09-20         None                   11   \n",
       "3072  WUR:CE-FIN05-IV-1036-A  2005-09-20         None                   22   \n",
       "3073  WUR:CE-FIN05-IV-1036-B  2005-09-20         None                   13   \n",
       "3074  WUR:CE-FIN05-IV-1036-C  2005-09-20         None                   14   \n",
       "3075  WUR:CE-FIN05-IV-1037-A  2005-09-20         None                   14   \n",
       "3076  WUR:CE-FIN05-IV-1037-B  2005-09-20         None                   12   \n",
       "3077  WUR:CE-FIN05-IV-1037-C  2005-09-20         None                    8   \n",
       "3078  WUR:CE-FIN05-IV-1038-A  2005-09-20         None                   18   \n",
       "3079  WUR:CE-FIN05-IV-1038-B  2005-09-20         None                    7   \n",
       "3080  WUR:CE-FIN05-IV-1038-C  2005-09-20         None                   11   \n",
       "3081  WUR:CE-FIN05-IV-1039-A  2005-09-20         None                    3   \n",
       "3082  WUR:CE-FIN05-IV-1039-B  2005-09-20         None                   13   \n",
       "3083  WUR:CE-FIN05-IV-1039-C  2005-09-20         None                    8   \n",
       "3084  WUR:CE-FIN05-IV-1040-A  2005-09-20         None                   16   \n",
       "3085  WUR:CE-FIN05-IV-1040-B  2005-09-20         None                   19   \n",
       "3086  WUR:CE-FIN05-IV-1040-C  2005-09-20         None                   22   \n",
       "\n",
       "     TubW_total_per_plant   DoE PlantHeight  \n",
       "0                    None    32           6  \n",
       "1                    None    27           8  \n",
       "2                    None    35           4  \n",
       "3                    None    25           7  \n",
       "4                    None  None           1  \n",
       "5                    None    26           8  \n",
       "6                    None    28           4  \n",
       "7                    None  None        None  \n",
       "8                    None    27           6  \n",
       "9                    None    24          11  \n",
       "10                   None    24          14  \n",
       "11                   None    26          11  \n",
       "12                   None    26          10  \n",
       "13                   None    26          16  \n",
       "14                   None    30           7  \n",
       "15                   None    26          12  \n",
       "16                   None    33           5  \n",
       "17                   None    34           3  \n",
       "18                   None    26           8  \n",
       "19                   None    26           8  \n",
       "20                   None    29           5  \n",
       "21                   None    28          10  \n",
       "22                   None    26          14  \n",
       "23                   None    26          15  \n",
       "24                   None    32           1  \n",
       "25                   None    28           9  \n",
       "26                   None    25          12  \n",
       "27                   None    26          11  \n",
       "28                   None    25          10  \n",
       "29                   None    28           7  \n",
       "...                   ...   ...         ...  \n",
       "3057               2667.5    18        None  \n",
       "3058                 1148    18        None  \n",
       "3059                 1096    18        None  \n",
       "3060                 1898    18        None  \n",
       "3061                 1808    18        None  \n",
       "3062                 1345    21        None  \n",
       "3063               1361.5    23        None  \n",
       "3064                  797    22        None  \n",
       "3065               1287.5    21        None  \n",
       "3066               1771.5    18        None  \n",
       "3067                 1674    18        None  \n",
       "3068                 2056    18        None  \n",
       "3069               1009.5    20        None  \n",
       "3070                605.5    22        None  \n",
       "3071               1191.5    21        None  \n",
       "3072                 1004    26        None  \n",
       "3073                 1060    25        None  \n",
       "3074                 1002    27        None  \n",
       "3075                  525    35        None  \n",
       "3076                  612    35        None  \n",
       "3077                428.5    35        None  \n",
       "3078                577.5    32        None  \n",
       "3079                524.5    28        None  \n",
       "3080                  545    34        None  \n",
       "3081                202.5    36        None  \n",
       "3082                  657    30        None  \n",
       "3083                  445    33        None  \n",
       "3084                643.5    33        None  \n",
       "3085                688.5    31        None  \n",
       "3086                  964    33        None  \n",
       "\n",
       "[3087 rows x 7 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2005Fin =  pd.read_csv(data_or + 'data_2005Fin.csv', sep='\\t')\n",
    "df_2005Fin = df_2005Fin.where(pd.notnull(df_2005Fin), None)\n",
    "df_2005Fin['Date']= pd.to_datetime(df_2005Fin['Date']).dt.date\n",
    "df_2005Fin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3087 entries, 0 to 3086\n",
      "Data columns (total 7 columns):\n",
      "Observation unit ID     3087 non-null object\n",
      "Date                    3087 non-null object\n",
      "NumMainStems            766 non-null object\n",
      "TubN_total_per_plant    1965 non-null object\n",
      "TubW_total_per_plant    1961 non-null object\n",
      "DoE                     3003 non-null object\n",
      "PlantHeight             2254 non-null object\n",
      "dtypes: object(7)\n",
      "memory usage: 84.4+ KB\n",
      "Are there any missing values?  True\n",
      "Are there any missing values in the Date column?  False\n",
      "How many missing values are there per column?\n",
      " Observation unit ID        0\n",
      "Date                       0\n",
      "NumMainStems            2321\n",
      "TubN_total_per_plant    1122\n",
      "TubW_total_per_plant    1126\n",
      "DoE                       84\n",
      "PlantHeight              833\n",
      "dtype: int64\n",
      "How many missing values are there overall?  5486\n",
      "Total cells: 21609\n",
      "Existing values: 16123\n",
      "Missing value percentage: 25.39%\n"
     ]
    }
   ],
   "source": [
    "# look at data info\n",
    "df_2005Fin;\n",
    "df_2005Fin.dtypes;\n",
    "df_2005Fin.info();\n",
    "# with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also\n",
    "#    print(df_2004FinNL)\n",
    "\n",
    "# examining missing values\n",
    "print('Are there any missing values? ', df_2005Fin.isnull().values.any())\n",
    "print('Are there any missing values in the Date column? ',df_2005Fin['Date'].isnull().values.any())\n",
    "print('How many missing values are there per column?\\n', df_2005Fin.isnull().sum())\n",
    "missing = df_2005Fin.isnull().sum().sum()\n",
    "print('How many missing values are there overall? ', missing)\n",
    "total_cells = len(df_2005Fin.columns) * len(df_2005Fin)\n",
    "print('Total cells: ' + str(total_cells))\n",
    "print('Existing values: ' + str(total_cells - missing))\n",
    "print('Missing value percentage: ' + str( \n",
    "      round(missing * 100 / total_cells,2)) + '%')\n",
    "\n",
    "# g.remove((None, None, None))\n",
    "g = parseStudyData('data_2005Fin.csv', df_2005Fin, nodes)\n",
    "#print(g_2004Fin.serialize(format='ttl').decode('utf8'))\n",
    "g.serialize(destination=data_gen + 'data_2005Fin.ttl', format='ttl');"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 37_32",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}