Update dictionary

README.md

@@ -17,7 +17,7 @@ The tables in the database are:
     ├────────────┼───────┼─────────┤
     │ projects   │     3 │       3 │
     │ sources    │    12 │       7 │
-    │ variables  │    85 │      10 │
+    │ variables  │   118 │      10 │
     │ vocab      │     1 │       6 │
     └────────────┴───────┴─────────┘
 

Snakefile

@@ -1,20 +1,43 @@
 from dotenv import load_dotenv
 import os
+import subprocess
 load_dotenv()
 
+def needs_update(flag):
+    update = subprocess.run(['bash', 'scripts/compare_dates.sh', flag], capture_output=True)
+    update = bool(int(update.stdout))
+    if update:
+        print(f'Flag {flag} needs to be updated')
+    else:
+        print(f'Flag {flag} is up to date')
+    return update
+
 tables = ['variables', 'sources', 'projects', 'vocab']
+
 envvars:
     'AIRTABLE_API_KEY',
     'MOTHERDUCK_TOKEN'
 
 conda:
     'environment.yml'
 
-rule check_dates:
+# rule check_dates:
+#     output:
+#         flag = touch('.needs_update'),
+#     shell:
+#         'bash scripts/compare_dates.sh .last_build .needs_update'
+rule check_update:
     output:
-        flag = touch('.needs_update'),
-    shell:
-        'bash scripts/compare_dates.sh .last_build .needs_update'
+        flag = '.needs_update',
+    run:
+        output_dir = os.path.dirname(output.flag)
+        if output_dir and not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        if needs_update('.last_build'):
+            shell('touch {output.flag}')
+        else:
+            shell('rm -f {output.flag}')
+
 
 rule download_data:
     params:
@@ -73,10 +96,9 @@ rule readme:
 
 rule all:
     input:
-        # rules.download_data.input.flag,
         '.needs_update',
         rules.gh_release.output.flag,
-        rules.md_upload.output.flag,
+        # rules.md_upload.output.flag,
         rules.readme.output.md,
         rules.build_db.output.db,
     default_target: True

TODO.md

@@ -0,0 +1,3 @@
+-[x] check against date in gh issue
+-[ ] programmatically get db name for artifact step
+

output_data/dictionary.json

@@ -0,0 +1 @@
+[{"source":"General terms","variables":[{"term":"Council of governments (COG)","definition":"Connecticut's towns are organized into planning regions, each served by a council of governments (COG) made up of mayors and first selectmen from the region. Starting in 2023, COGs began replacing previous county designations for federal and state statistics. ","url":"https://portal.ct.gov/OPM/IGPP/ORG/Planning-Regions/Planning-Regions---Overview"}]},{"source":"US Census Bureau American Community Survey 5-year estimates","variables":[{"term":"Latino population","definition":"In Census data, Latino is considered an ethnicity, not a racial category. The Latino population here is anyone of Latino ethnicity, regardless of race."},{"term":"White population","definition":"The white population here represents people who are white and not of Latino ethnicity."},{"term":"Black population","definition":"The Black population here represents people who are Black and not of Latino ethnicity."},{"term":"Other race population","definition":"The \"other race\" population here represents people who are not in the white or Black racial groups and are not of Latino ethnicity."},{"term":"Foreign-born population","definition":"In Census data, people are considered native-born if they were born in the US or its territories, or born abroad to US citizen parents. All other US residents are considered foreign-born."},{"term":"Cost-burdened households","definition":"Based on HUD guidelines, a household is considered cost-burdened if more than 30% of its income is spent on housing costs."},{"term":"Poverty status known","definition":"Because poverty status is based on total household income and the number of people in that household, not everyone's poverty status can be calculated. This variable gives the number of people in the area for whom poverty status can be calculated, and is the denominator for poverty and low-income rates."},{"term":"Population in poverty","definition":"Counts of people in poverty and poverty rates represent the number and percentage of people whose household incomes are below the federal poverty level."},{"term":"Low-income population","definition":"Counts of low-income people and low-income rates represent the number and percentage of people whose household incomes are below two times the federal poverty level."}]},{"source":"Questions on the DataHaven Community Wellbeing Survey","variables":[{"term":"Just getting by financially","definition":"How well would you say you are managing financially these days? (just getting by / finding it difficult / finding it very difficult)"},{"term":"Food insecure in past year","definition":"Have there been times in the past 12 months when you did not have enough money to buy food that you or your family needed?"},{"term":"Housing insecure in past year","definition":"In the last 12 months, have you not had enough money to provide adequate shelter or housing for you or your family?"},{"term":"Transportation insecure in past year","definition":"In the past 12 months, did you stay home when you needed or wanted to go someplace because you had no access to reliable transportation?"},{"term":"Access to a car","definition":"Do you have access to a car when you need it? (very often / fairly often)"},{"term":"Stores, banks, etc. are in walking distance","definition":"Many stores, banks, markets or places to go are within easy walking distance of my home (strongly agree / agree)"},{"term":"Safe places to bike in area","definition":"There are places to bicycle in or near my neighborhood that are safe from traffic, such as on the street or on special lanes, separate paths or trails (strongly agree / agree)"},{"term":"Local recreational facilities available","definition":"My neighborhood has several free or low cost recreation facilities such as parks, playgrounds, public swimming pools, etc (strongly agree / agree)"},{"term":"Feel safe walking at night","definition":"I do not feel safe to go on walks in my neighborhood at night (strongly disagree / disagree)"},{"term":"Trust neighbors","definition":"People in this neighborhood can be trusted (strongly agree / agree)"},{"term":"Local youth have positive role models","definition":"Children and youth in my town generally have the positive role models they need around here (strongly agree / agree)"},{"term":"Have some influence over local government","definition":"How would you describe your ability to influence local-government decision making? (great influence / moderate influence / a little influence)"},{"term":"Health is excellent or very good","definition":"How would you rate your overall health? (excellent / very good)"},{"term":"Satisfied with life","definition":"How satisfied are you with your life nowadays? (mostly / completely)"},{"term":"Mostly or completely happy","definition":"How happy did you feel yesterday? (mostly / completely)"},{"term":"Mostly or completely anxious","definition":"How anxious did you feel yesterday? (mostly / completely)"},{"term":"Have social and emotional support","definition":"How often do you get the social and emotional support you need? (always / usually)"},{"term":"Satisfied with area","definition":"Are you satisfied with the city or area where you live?"},{"term":"Local government is responsive","definition":"How responsive local government is to the needs of residents (excellent / good)"},{"term":"Local police keep residents safe","definition":"The job done by the police to keep residents safe (excellent / good)"},{"term":"Suitable work opportunities available","definition":"The ability of residents to obtain suitable employment (excellent / good)"},{"term":"Area is good to raise kids","definition":"As a place to raise children (excellent / good)"},{"term":"Parks are in good condition","definition":"The condition of public parks and other public recreational facilities (excellent / good)"},{"term":"Affordable quality produce is available","definition":"The availability of affordable, high-quality fruits and vegetables (excellent / good)"}]}]

scripts/compare_dates.sh

@@ -4,7 +4,7 @@
 # return 1 (update) if airtable date is later than flag file date, 0 (don't update) otherwise.
 # if no date flag exists, return 1.
 flagin=$1
-flagout=$2
+# flagout=$2
 need_update=1
 airtable=$(gh issue view 1 --json body --jq '.body')
 
@@ -17,8 +17,8 @@ if [ -f "$flagin" ]; then
 fi
 
 # if need_update = 1, touch flagfile
-if [ "$need_update" -eq 1 ]; then
-    touch "$flagout"
-fi
+# if [ "$need_update" -eq 1 ]; then
+#     touch "$flagout"
+# fi
 
 echo "$need_update"