let's see if this works...

chatbot has multiple datasets now. Also, pmtiles only shows up in notebook-- not sure if this will work

.gitignore

@@ -9,3 +9,8 @@
 *.vrt
 .streamlit
 __pycache__
+*.block
+*.tmp
+*.parquet
+*.geojson
+**/*.shp

app/app.py

@@ -4,13 +4,28 @@ from cng.h3 import *
 from utils import *
 from ibis import _
 
-current_tables = con.list_tables()
+# current_tables = con.list_tables()
 
-if "mydata" not in set(current_tables):
-    con.create_table("mydata", database_geom)
+# if "conservation_almanac" not in set(current_tables):
+#     con.create_table("conservation_almanac", tpl_z8)
 
-chatbot_data = con.table("mydata")
+# if "landvote" not in set(current_tables):
+#     con.create_table("landvote", landvote_z8)
 
+# if "carbon" not in set(current_tables):
+#     con.create_table("carbon", carbon_z8)
+
+# if "mobi" not in set(current_tables):
+#     con.create_table("mobi", mobi_z8)
+
+# if "svi" not in set(current_tables):
+#     con.create_table("svi", svi_z8)
+
+# conservation_almanac = con.table("conservation_almanac")
+# landvote = con.table("landvote")
+# carbon = con.table("carbon")
+# mobi = con.table("mobi")
+# svi = con.table("svi")
 
 st.set_page_config(layout="wide",
                    page_title="TPL Conservation Almanac",
@@ -25,7 +40,6 @@ basemaps = leafmap.basemaps.keys()
 
 m = leafmap.Map(style = "positron")
 
-
 from datetime import time
 
 with st.sidebar:
@@ -93,7 +107,9 @@ from langchain_core.prompts import ChatPromptTemplate
 prompt = ChatPromptTemplate.from_messages([
     ("system", template),
     ("human", "{input}")
-]).partial(dialect="duckdb", table_info = chatbot_data.schema())
+]).partial(dialect="duckdb", conservation_almanac = tpl_z8.schema(),
+          landvote = landvote_z8.schema(), carbon = carbon_z8.schema(),
+          svi = svi_z8.schema(), mobi = mobi_z8.schema())
 
 # chatbot_toggles = {key: False for key in keys}
 structured_llm = llm.with_structured_output(SQLResponse)
@@ -115,7 +131,7 @@ def run_sql(query,paint):
         st.success(explanation)
         return pd.DataFrame({'fid' : []})
         
-    result = chatbot_data.sql(sql_query).distinct().execute()
+    result = con.sql(sql_query).distinct().execute()
     if result.empty :
         explanation = "This query did not return any results. Please try again with a different query."
         st.warning(explanation, icon="⚠️")
@@ -127,9 +143,6 @@ def run_sql(query,paint):
             return result
     elif ("fid" and "geom" in result.columns): 
         style = tpl_style(result["fid"].tolist(), paint)
-        # legend, position, bg_color, fontsize = get_legend(paint)
-    
-        # m.add_legend(legend_dict = legend, position = position, bg_color = bg_color, fontsize = fontsize)
         m.add_pmtiles(pmtiles, style=style, opacity=0.5, tooltip=True, fit_bounds=True)
         m.fit_bounds(result.total_bounds.tolist())    
         result = result.drop('geom',axis = 1) #printing to streamlit so I need to drop geom
@@ -180,14 +193,8 @@ if 'out' not in locals():
         m.add_pmtiles(pmtiles, style=tpl_style(unique_ids, paint), opacity=0.5, tooltip=True, fit_bounds=True)
     else:        
         m.add_pmtiles(pmtiles, style=tpl_style_default(paint), opacity=0.5, tooltip=True, fit_bounds=True)
-
-    # legend, position, bg_color, fontsize = get_legend(paint)
-    # m.add_legend(legend_dict = legend, position = position, bg_color = bg_color, fontsize = fontsize)
-    #zoom to state(s)
     fit_bounds(state_choice, county_choice, m)
 
-## Render display panels
-# 
 
 m.to_streamlit()
 with st.expander("🔍 View/download data"): # adding data table  
@@ -216,7 +223,7 @@ with col1:
     get_bar(gdf_tpl, style_choice, 'year', 'total_amount', paint,'Year','Acquisition Cost ($)',"Yearly investment ($) in protected area")
 
 with col2:
-    gdf_landvote = group_data(gdf_landvote.filter(_.measure_status == 'Pass'), 'Measure Cost')
+    gdf_landvote = group_data(gdf_landvote.filter(_.status == 'Pass'), 'Measure Cost')
     get_bar(gdf_landvote, style_choice, 'year', 'total_amount', paint, 'Year','Funds Approved ($)','Yearly funds from conservation ballot measures')
 
 st.divider()

app/system_prompt.txt

@@ -1,4 +1,27 @@
-You are an expert in SQL and an assistant for mapping and analyzing the Trust for Public Land (TPL) data. The dataset is synthesized from TPL Conservation Alamanc, which track the impacts of public spending on land conservation, and TPL Landvote, a comprehensive database of conservation ballot measures. Given an input question, create a syntactically correct {dialect} query to run, and then provide an explanation of how you answered the input question. If the question doesn't necessitate a SQL query, only output an explanation.
+You are an expert in SQL and an assistant for mapping and analyzing the Trust for Public Land (TPL) data.  You are provided multiple tables and must identify which table(s) to use. Given an input question, create a syntactically correct {dialect} query to run, and then provide an explanation of how you answered the input question. Not every query will require SQL code, users may ask more information about values and columns in the table which you can answer based on the information in this prompt. For these cases, your "sql_query" field should be empty.  
+
+ONLY write SQL queries using the records and columns that exist in the relevant table. You have access to these tables:
+
+conservation_almanac: 
+- Definition: Protected areas database tracking public spending on land conservation. 
+- Schema: {conservation_almanac}
+
+landvote: 
+- Definition: Tracks land conservation ballot measures.
+- Schema: {landvote}
+
+carbon: 
+- Definition: level of irrecoverable carbon
+- Schema: {carbon}
+
+mobi: 
+- Definition: species richness from the NatureServe's Map of Biodiversity Importance (MOBI)
+- Schema: {mobi}
+
+svi: 
+- Definition: social vulnerability index; higher value indicates higher vulnerability. Make sure to only use svi values greater than 0. 
+- Schema: {svi}
+
 
 For example:
 {{
@@ -8,11 +31,9 @@ For example:
 
 Ensure the response contains only this JSON object, with no additional text, formatting, or commentary.
 
-ONLY write SQL queries using the records and columns that exist in the table called mydata. This is the schema for mydata: {table_info}.
-
 # Important Details
-     - Not every query will require SQL code, users may ask more information about values and columns in the table which you can answer based on the information in this prompt. For these cases, your "sql_query" field should be empty.  
-    - For map-related queries (e.g., "show me"), ALWAYS include "fid", "Site_Name" ,"geom", and "acres" in the results, 
+    - When joining tables, use the `h8` column.
+    - For visualization-related queries (e.g., "show me"), ALWAYS include "fid", "Site_Name" in the results, 
     - ONLY use LIMIT in your SQL queries if the user specifies a quantity (e.g., 'show me 5'). Otherwise, return all matching data without a limit.
     - Wrap each column name in double quotes (") to denote them as delimited identifiers.
     - Wrap values that are strings in single quotes (') to distinguish them from column names. 
@@ -22,17 +43,40 @@ ONLY write SQL queries using the records and columns that exist in the table cal
 ## Example:
 example_user: "What is most expensive protected site?"
 example_assistant: {{"sql_query": 
-    SELECT "fid", "Site_Name" ,"geom", "acres", "Amount"
-    FROM mydata 
-    WHERE "Amount" = (SELECT MAX("Amount") FROM mydata);
+    SELECT "fid", "Site_Name", "Amount"
+    FROM conservation_almanac 
+    WHERE "Amount" = (SELECT MAX("Amount") FROM conservation_almanac);
 "explanation":"I selected the site with the highest `Amount`.
 }}
 
 ## Example:
 example_user: "Which sites are owned, managed or sponsored by the Trust for Public Land?"
 example_assistant: {{"sql_query": 
-    SELECT DISTINCT "fid", "Site_Name", "geom", "acres", "Owner_Name", "Manager_Name", "Sponsor_Name" FROM mydata 
+    SELECT DISTINCT "fid", "Site_Name", "Owner_Name", "Manager_Name", "Sponsor_Name" FROM conservation_almanac 
     WHERE "Owner_Name" ILIKE '%Trust for Public Land%' OR "Manager_Name" ILIKE '%Trust for Public Land%' OR "Sponsor_Name" ILIKE '%Trust for Public Land%' 
     ORDER BY "Site_Name";
 "explanation":"I selected all sites affiliated with the Trust for Public Land.
 }}
+
+## Example:
+example_user: "Show me protected areas with high levels of carbon"
+example_assistant: {{"sql_query": 
+    SELECT fid, AVG(carbon) as mean_carbon
+    FROM conservation_almanac
+    LEFT JOIN carbon
+    USING (h8)
+    GROUP BY fid
+    ORDER BY mean_carbon DESC LIMIT 10;
+"explanation":"I joined `conservation_almanac` with `carbon` to retrieve irrecoverable carbon levels for protected areas in the `conservation_almanac`. I returned 10 areas with highest levels of carbon.  
+}}
+
+example_user: "Show me protected areas that are in socially vulnerable areas  
+example_assistant: {{"sql_query": 
+    SELECT fid, AVG(carbon) as mean_carbon
+    FROM conservation_almanac
+    LEFT JOIN carbon
+    USING (h8)
+    GROUP BY fid
+    ORDER BY mean_carbon DESC LIMIT 10;
+"explanation":"I joined `conservation_almanac` with `carbon` to retrieve irrecoverable carbon levels for protected areas in the `conservation_almanac`. I returned 10 areas with highest levels of carbon.  
+}}

app/utils.py

@@ -5,22 +5,23 @@ import altair as alt
 import re
 
 def get_counties(state_selection):
+    tpl_table.head().execute()
+
     if state_selection != 'All':
-        counties = database.filter(_.state_name == state_selection).select('county').distinct().order_by('county').execute()
+        counties = tpl_table.filter(_.state == state_selection).select('county').distinct().order_by('county').execute()
         counties = ['All'] + counties['county'].tolist()
     else: 
         counties = None
     return counties
      
 def filter_data(table, state_choice, county_choice, year_range):
+
     min_year, max_year = year_range
     gdf = (table.filter(_.year>=(min_year))
            .filter(_.year<=(max_year))
           )
     if state_choice != "All":
-        # gdf = gdf.filter(_.state_name.isin(state_choice))
-        gdf = gdf.filter(_.state_name == state_choice)
-
+        gdf = gdf.filter(_.state == state_choice)
         if (county_choice != "All") and (county_choice):
             county_choice = re.sub(r"(?i)\s*(County)\b", "", county_choice)    
             gdf = gdf.filter(_.county == county_choice)
@@ -34,8 +35,8 @@ def group_data(table, style_choice):
 
 def fit_bounds(state_choice, county_choice, m):
     if state_choice != "All":
-        # gdf = county_bounds.filter(_.state_name.isin(state_choice))
-        gdf = county_bounds.filter(_.state_name == state_choice)
+        # gdf = county_bounds.filter(_.state.isin(state_choice))
+        gdf = county_bounds.filter(_.state == state_choice)
 
         if (county_choice != "All") and (county_choice):
             gdf = gdf.filter(_.county == county_choice)
@@ -105,7 +106,8 @@ def tpl_style(ids, paint):
             "source": "tpl",
             "source-layer": source_layer_name,
             "type": "fill",
-            'filter': ['in', ['get', 'fid'], ["literal", ids]],
+            'filter': ["match", ["get", 'fid'], ids, True, False],
+            # 'filter': ['in', ['get', 'fid'], ["literal", ids]],
             "paint": {
                 "fill-color": paint,
                 "fill-opacity": 1
@@ -129,28 +131,26 @@ def get_legend(paint):
 
 @st.cache_data
 def tpl_summary(_df):
-    summary = _df.group_by(_.Manager_Type).agg(Amount = _.Amount.sum())
-    public_dollars = round( summary.filter(_.Manager_Type.isin(["FED", "STAT", "LOC", "DIST"])).agg(total = _.Amount.sum()).to_pandas().values[0][0] )
-    private_dollars = round( summary.filter(_.Manager_Type.isin(["PVT", "NGO"])).agg(total = _.Amount.sum()).to_pandas().values[0][0] )
-    # tribal_dollars = summary.filter(_.Manager_Type.isin(["TRIB"])).agg(total = _.Amount.sum()).to_pandas().values[0][0] 
-    # tribal_dollars = tribal_dollars if tribal_dollars else round(tribal_dollars)
-    total_dollars = round( summary.agg(total = _.Amount.sum()).to_pandas().values[0][0] )
+    summary = _df.group_by(_.manager_type).agg(amount = _.amount.sum())
+    public_dollars = round( summary.filter(_.manager_type.isin(["FED", "STAT", "LOC", "DIST"])).agg(total = _.amount.sum()).to_pandas().values[0][0] )
+    private_dollars = round( summary.filter(_.manager_type.isin(["PVT", "NGO"])).agg(total = _.amount.sum()).to_pandas().values[0][0] )
+    total_dollars = round( summary.agg(total = _.amount.sum()).to_pandas().values[0][0] )
     return public_dollars, private_dollars, total_dollars
 
 # @st.cache_data
 def calc_delta(_df):
     deltas = (_df
-     .group_by(_.Manager_Type, _.year)
-     .agg(Amount = _.Amount.sum())
-     .mutate(total = _.Amount.cumsum(order_by=_.year, group_by=_.Manager_Type))
+     .group_by(_.manager_type, _.year)
+     .agg(amount = _.amount.sum())
+     .mutate(total = _.amount.cumsum(order_by=_.year, group_by=_.manager_type))
      .mutate(lag = _.total.lag(1))
      .mutate(delta = (100*(_.total - _.lag) / _.total).round(2)  )
      # .filter(_.year >=2019)
-     .select(_.Manager_Type, _.year, _.total, _.lag, _.delta)
+     .select(_.manager_type, _.year, _.total, _.lag, _.delta)
     )
-    public_delta = deltas.filter(_.Manager_Type.isin(["FED", "STAT", "LOC", "DIST"])).to_pandas()
+    public_delta = deltas.filter(_.manager_type.isin(["FED", "STAT", "LOC", "DIST"])).to_pandas()
     public_delta =  0 if public_delta.empty else public_delta.delta[-1]
-    private_delta = deltas.filter(_.Manager_Type.isin(["PVT", "NGO"])).to_pandas()
+    private_delta = deltas.filter(_.manager_type.isin(["PVT", "NGO"])).to_pandas()
     private_delta =  0 if private_delta.empty else private_delta.delta[-1]
     return public_delta, private_delta
     
@@ -180,9 +180,9 @@ def bar(area_totals, column, paint):
 #         .filter(~_.year.isnull())
 #         .filter(_.year > 0)
 #         .group_by([_.year, _[column]])
-#         .agg(Amount = _.Amount.sum())
+#         .agg(amount = _.amount.sum())
 #         .mutate(year = _.year.cast("int"),
-#                 Amount = _.Amount.cumsum(group_by=_[column], order_by=_.year))
+#                 amount = _.amount.cumsum(group_by=_[column], order_by=_.year))
         
 #         .to_pandas()
 #     )
@@ -196,7 +196,7 @@ def chart_time(timeseries, column, paint):
     # use the colors 
     plt = alt.Chart(timeseries).mark_line().encode(
         x='year:O',
-        y = alt.Y('Amount:Q'),
+        y = alt.Y('amount:Q'),
             color=alt.Color(column,scale= alt.Scale(domain=domain, range=range_))
     ).properties(height=350)
     return plt

app/variables.py

@@ -10,8 +10,8 @@ from datetime import timedelta
 import re
 duckdb_install_h3()
 
-# con = ibis.duckdb.connect("duck.db",extensions = ["spatial", "h3"])
 con = ibis.duckdb.connect(extensions = ["spatial", "h3"])
+con.raw_sql("SET THREADS=100;")
 set_secrets(con)
 
 # Get signed URLs to access license-controlled layers
@@ -19,72 +19,30 @@ key = st.secrets["MINIO_KEY"]
 secret = st.secrets["MINIO_SECRET"]
 client = Minio("minio.carlboettiger.info", key, secret)
 
-mobi = con.read_parquet("https://minio.carlboettiger.info/public-mobi/hex/all-richness-h8.parquet").select("h8", "Z").rename(richness = "Z")
-svi = con.read_parquet("https://minio.carlboettiger.info/public-social-vulnerability/2022/SVI2022_US_tract_h3_z8.parquet").select("h8", "svi").filter(_.svi > 0)
-# carbon = con.read_parquet("https://minio.carlboettiger.info/public-carbon/hex/us-tracts-vuln-total-carbon-2018-h8.parquet").select('carbon','h8')
-
-tpl_geom_url = "s3://shared-tpl/tpl.parquet"
-tpl_table = con.read_parquet(tpl_geom_url).mutate(geom = _.geom.convert("ESRI:102039", "EPSG:4326")).rename(year = 'Close_Year', state_name = 'State', county = 'County')
+tpl_z8 = con.read_parquet("s3://shared-tpl/conservation_almanac/z8/tpl_h3_z8.parquet", table_name = 'conservation_almanac')
+landvote_z8 = con.read_parquet("s3://shared-tpl/landvote/z8/landvote_h3_z8.parquet", table_name = 'landvote')
+mobi_z8 = con.read_parquet("https://minio.carlboettiger.info/public-mobi/hex/all-richness-h8.parquet", table_name = 'mobi')
+svi_z8 = con.read_parquet("https://minio.carlboettiger.info/public-social-vulnerability/2022/SVI2022_US_tract_h3_z8.parquet",table_name = 'svi')
+carbon_z8 = con.read_parquet("https://minio.carlboettiger.info/public-carbon/hex/us-tracts-vuln-total-carbon-2018-h8.parquet",table_name = 'carbon')
 
 county_bounds = con.read_parquet("https://minio.carlboettiger.info/public-census/2024/county/2024_us_county.parquet")
-
-landvote_z8 = (con.read_parquet("s3://shared-tpl/landvote_h3_z8.parquet")
-            .rename(FIPS_county = "FIPS", measure_amount = 'Conservation Funds Approved', 
-                    measure_status = "Status", measure_purpose = "Purpose",)
-            .mutate(measure_year = _.Date.year()).drop('Date','geom'))
-
-
-landvote_table = (con.read_parquet("s3://shared-tpl/landvote_geom.parquet")
-            .rename(FIPS_county = "FIPS", measure_amount = 'Conservation Funds Approved', 
-                    measure_status = "Status", measure_purpose = "Purpose")
-            .mutate(year = _.Date.year()).drop('Date'))
-
-
-tpl_drop_cols = ['Reported_Acres','Close_Date','EasementHolder_Name',
-        'Data_Provider','Data_Source','Data_Aggregator',
-        'Program_ID','Sponsor_ID']
-tpl_z8_url = "s3://shared-tpl/tpl_h3_z8.parquet"
-tpl_z8 = con.read_parquet(tpl_z8_url).mutate(h8 = _.h8.lower()).drop(tpl_drop_cols)
-
-select_cols = ['fid','TPL_ID','landvote_id',
-'state','state_name','county',
- 'FIPS_county',
- 'city','jurisdiction',
- 'Close_Year', 'Site_Name',
- 'Owner_Name','Owner_Type',
- 'Manager_Name','Manager_Type',
- 'Purchase_Type','EasementHolder_Type',
- 'Public_Access_Type','Purpose_Type',
- 'Duration_Type','Amount',
- 'Program_Name','Sponsor_Name',
- 'Sponsor_Type','measure_year',
- 'measure_status','measure_purpose',
- 'measure_amount',
- # 'carbon',
- 'richness','svi',
- 'h8']
-
-database = (
-  tpl_z8.drop('State','County')
-  .left_join(landvote_z8, "h8").drop('h8_right')
-  .left_join(svi, "h8").drop('h8_right')
-  .left_join(mobi, "h8").drop('h8_right')
-  # .left_join(carbon, "h8").drop('h8_right')
-).select(select_cols).distinct()
-
-database_geom = (database.drop('h8').distinct().inner_join(tpl_table.select('geom','TPL_ID','fid','Shape_Area'), [database.fid == tpl_table.fid])
-            .mutate(acres = _.Shape_Area*0.0002471054)
-           )
+landvote_table = con.read_parquet("s3://shared-tpl/landvote/landvote_geom.parquet")
+tpl_table = con.read_parquet('s3://shared-tpl/conservation_almanac/tpl.parquet')
 
 pmtiles = client.get_presigned_url(
     "GET",
     "shared-tpl",
-    "tpl_v2.pmtiles",
+    "conservation_almanac/tpl.pmtiles",
     expires=timedelta(hours=2),
 )
+# pmtiles = client.get_presigned_url(
+#     "GET",
+#     "shared-tpl",
+#     "tpl_v2.pmtiles",
+#     expires=timedelta(hours=2),
+# )
 
-source_layer_name = 'tpl'
-# source_layer_name = re.sub(r'\W+', '', os.path.splitext(os.path.basename(pmtiles))[0]) #stripping hyphens to get layer name 
+source_layer_name = re.sub(r'\W+', '', os.path.splitext(os.path.basename(pmtiles))[0]) #stripping hyphens to get layer name 
 
 states = (
     "All", "Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", "Connecticut",
@@ -116,7 +74,7 @@ style_options = {
     "Acquisition Cost":  
             ["interpolate",
                 ['exponential', 1], 
-                ["get", "Amount"],
+                ["get", "amount"],
                     0,	"#fde725",
                     36000,	"#b4de2c",
                     93000,	"#6ccd59",
@@ -130,7 +88,7 @@ style_options = {
                     ] 
             ,
     "Manager Type":  {
-            'property': 'Manager_Type',
+            'property': 'manager_type',
             'type': 'categorical',
             'stops': [
                 ['FED', darkblue],
@@ -145,7 +103,7 @@ style_options = {
             ]
             },
     "Access": {
-        'property': 'Public_Access_Type',
+        'property': 'access_type',
         'type': 'categorical',
         'stops': [
             ['OA', green],
@@ -155,7 +113,7 @@ style_options = {
         ]
     },
     "Purpose": {
-        'property': 'Purpose_Type',
+        'property': 'purpose_type',
         'type': 'categorical',
         'stops': [
             ['FOR', green],
@@ -174,12 +132,10 @@ style_options = {
 style_choice_columns = {'Manager Type': style_options['Manager Type']['property'],
               'Access' : style_options['Access']['property'],
               'Purpose': style_options['Purpose']['property'],
-                'Acquisition Cost': 'Amount',
-                'Measure Cost': 'measure_amount',
+                'Acquisition Cost': 'amount',
+                'Measure Cost': 'conservation_funds_approved',
              }
 
-# metric_columns = {'svi': 'svi', 'mobi': 'richness', 'landvote':'measure_status'}
-
 from langchain_openai import ChatOpenAI
 import streamlit as st