EarthRanger IO#
Setup#
Ecoscope#
[ ]:
%pip install 'ecoscope[analysis,mapping,plotting] @ git+https://github.com/wildlife-dynamics/ecoscope@v1.8.8' &> /dev/null
[ ]:
import getpass
import os
import sys
import geopandas as gpd
import pandas as pd
from erclient.client import ERClientException
from shapely.geometry import Point
import ecoscope
ecoscope.init()
Google Drive Setup#
[ ]:
output_dir = "Ecoscope-Outputs"
if "google.colab" in sys.modules:
from google.colab import drive
drive.mount("/content/drive/", force_remount=True)
output_dir = os.path.join("/content/drive/MyDrive/", output_dir)
os.makedirs(output_dir, exist_ok=True)
Connect to EarthRanger#
[ ]:
ER_SERVER = os.getenv("ER_SERVER", "https://mep-dev.pamdas.org")
ER_USERNAME = os.getenv("ER_USERNAME", "username") # replace with your username
ER_PASSWORD = os.getenv("ER_PASSWORD", "")
if not ER_PASSWORD:
ER_PASSWORD = getpass.getpass("Please enter your ER password: ")
er_io = ecoscope.io.EarthRangerIO(
server=ER_SERVER,
username=ER_USERNAME,
password=ER_PASSWORD,
tcp_limit=5,
sub_page_size=4000,
)
GET Event
#
By event_ids
#
Use this approach to download an individual Event
if you know its id. Downloading from multiple ids at once is not currently supported in ER.
[ ]:
er_io.get_events(
event_ids="76e76c8c-0936-4ef4-9298-0cbd4f516a9c", # Distance count event
)
By event_type
#
Use this approach if you want to download events of a given event type (using the EventType ID from ER)
[ ]:
df_event_types = pd.DataFrame(er_io.get_event_types())
mepdev_distance_count_id = df_event_types[df_event_types["value"] == "mepdev_distance_count"]["id"].values[0]
mepdev_distance_count_id
[ ]:
er_io.get_events(
event_type=[mepdev_distance_count_id], # MEP Distance Count
since=pd.Timestamp("2023-01-01").isoformat(),
until=pd.Timestamp("2024-01-01").isoformat(),
).reset_index()
Unpack event_details
column of JSON
data into individual columns#
[ ]:
df_event_types = pd.DataFrame(er_io.get_event_types())
mepdev_distance_survey_id = df_event_types[df_event_types["value"] == "mepdev_distance_count"]["id"].values[0]
mepdev_distance_survey_id
[ ]:
events_df = er_io.get_events(event_type=[mepdev_distance_survey_id]) # MEP Distance Survey Count
events_df.event_details
[ ]:
ecoscope.io.earthranger_utils.normalize_column(events_df, "event_details")
events_df
POST Event
#
First delete any existing events:
[ ]:
df_event_types = pd.DataFrame(er_io.get_event_types())
carcass_id = df_event_types[df_event_types["value"] == "carcass_rep"]["id"].values[0]
carcass_id
[ ]:
try:
event_df = er_io.get_events(
event_type=[carcass_id], # Carcass
).reset_index()
for i, value in event_df.iterrows():
er_io.delete_event(event_id=event_df.loc[i, "id"])
except AssertionError:
print("No events exist yet")
Define events to be uploaded:
[ ]:
new_event = [
{
"event_details": {
"carcassrep_ageofanimal": None,
"carcassrep_ageofcarcass": None,
"carcassrep_causeofdeath": None,
"carcassrep_sex": None,
"carcassrep_species": None,
"carcassrep_trophystatus": None,
},
"event_type": "carcass_rep",
"icon_id": "carcass_rep",
"id": "e29f9078-ee0a-4f06-b685-92e9ff266e9b",
"location": {"latitude": -27.12759, "longitude": -109.40804},
"priority": 100,
"state": "resolved",
"time": pd.Timestamp("2022-03-15 15:44:00-0700"),
"title": "Carcass",
}
]
Read events as dataframe:
[ ]:
new_event_df = pd.DataFrame.from_dict(new_event)
new_event_df
Upload:
[ ]:
try:
er_io.post_event(events=new_event_df.iloc[[0]])
except ERClientException as e:
print(e)
PATCH Event
#
Define updated values
[ ]:
updated_event = pd.DataFrame(
[
{
"event_details": {
"carcassrep_ageofanimal": "adult",
"carcassrep_ageofcarcass": "less_than_a_day",
"carcassrep_causeofdeath": "unnaturalshot",
"carcassrep_sex": "female",
"carcassrep_species": "elephant",
"carcassrep_trophystatus": "removed",
}
}
]
)
Get and display current values
[ ]:
event_df = er_io.get_events(
event_type=["b0d66402-34cb-4e9a-9715-2ac4eeebb574"], # Carcass
).reset_index()
event_df.loc[0, "event_details"]
Patch and confirm values have been updated
[ ]:
id_to_patch = event_df.loc[0, "id"]
er_io.patch_event(event_id=id_to_patch, events=updated_event).loc[0, "event_details"]
GET Subjects
#
by id
#
[ ]:
df_subjects = er_io.get_subjects()
habiba_id = df_subjects[df_subjects["name"] == "Habiba"]["id"].values[0]
habiba_id
[ ]:
er_io.get_subjects(id=habiba_id) # Habiba
By SubjectGroup
subject_group_name
#
[ ]:
er_io.get_subjects(subject_group_name="Elephants")
GET Observations
#
The output is an Ecoscope Relocations
dataframe that has a core set of columns: groupby_col
, fixtime
, junk_status
.
All other columns are prepended by extra__
. The groupby_col
will be the subject_id
and the index of the dataframe is the observation indices.
Classic ER Observation
format can be returned by passing in parameter relocations=False
.
Filter observation by setting filter
to one of: None, 0, 1, 2 or 3 - None returns everything - 0 filters out everything but observations with exclusion flag 0 (Pass back clean data) - 1 filters out everything but observations with exclusion flag 1 (Pass back manually filtered data) - 2 filters out everything but observations with exclusion flag 2 (Ppass back automatically filtered data) - 3 filters out everything but observations with exclusion flag 2 or 1 (Pass back both manual and
automatic filtered data)
By SubjectGroup
subject_group_name
:#
[ ]:
since = pd.Timestamp("2008-01-01").isoformat()
until = pd.Timestamp("2020-01-01").isoformat()
[ ]:
relocs = er_io.get_subjectgroup_observations(
subject_group_name="Elephants",
filter=0,
since=since,
until=until,
include_details=True,
include_inactive=True,
include_source_details=True,
include_subject_details=True,
include_subjectsource_details=True,
)
relocs
By subject_id
#
[ ]:
df_subjects = er_io.get_subjects()
habiba_id = df_subjects[df_subjects["name"] == "Habiba"]["id"].values[0]
salif_keita_id = df_subjects[df_subjects["name"] == "Salif Keita"]["id"].values[0]
habiba_id, salif_keita_id
[ ]:
relocs = er_io.get_subject_observations(
subject_ids=[
habiba_id, # Habiba
salif_keita_id, # Salif Keita
],
since=since,
until=until,
filter=0,
include_details=True,
include_subject_details=True,
include_source_details=True,
include_subjectsource_details=True,
)
relocs
By source_id
#
[ ]:
df_sources = er_io.get_sources()
habiba_source_id = df_sources[df_sources["manufacturer_id"] == "ST-Habiba"]["id"].values[0]
salif_keita_source_id = df_sources[df_sources["manufacturer_id"] == "ST-SalifKeita"]["id"].values[0]
habiba_source_id, salif_keita_source_id
[ ]:
df_sources = er_io.get_sources()
relocs = er_io.get_source_observations(
source_ids=[
habiba_source_id, # Habiba's Source
salif_keita_source_id, # Salif Keita's Source
],
since=since,
until=until,
filter=0,
include_details=True,
include_source_details=True,
)
relocs
GET Patrols
#
[ ]:
df_patrol_types = er_io.get_patrol_types().reset_index()
mep_distance_id = df_patrol_types[df_patrol_types["value"] == "MEP_Distance_Survey_Patrol"]["id"].values[0]
mep_distance_id
[ ]:
er_io.get_patrols(patrol_type=[mep_distance_id]) # MEP Distance Count Survey
GET Observations
for a Patrol
#
[ ]:
patrol_df = er_io.get_patrols(
since=pd.Timestamp("2017-01-01").isoformat(),
until=pd.Timestamp("2017-04-01").isoformat(),
)
relocs = er_io.get_patrol_observations(
patrol_df,
include_patrol_details=True,
include_source_details=True,
include_subject_details=True,
include_subjectsource_details=True,
)
relocs
GET SPATIAL FEATURES
#
GET SPATIAL FEATURE
from spatial_feature_id
#
[ ]:
spatial_feature = er_io.get_spatial_feature(spatial_feature_id="8868718f-0154-45bf-a74d-a66706ef958f")
spatial_feature
GET SPATIAL FEATURES
from spatial_features_group_id
#
[ ]:
spatial_features = er_io.get_spatial_features_group(spatial_features_group_id="15698426-7e0f-41df-9bc3-495d87e2e097")
spatial_features
GET Users
#
[ ]:
df = pd.DataFrame(er_io.get_users())
df
POST Observation
#
First delete any existing observations
[ ]:
elephants = er_io.get_subjectgroup_observations(
subject_group_name="Elephants", since=pd.Timestamp("2022-01-01").isoformat()
)
if not elephants.empty:
for observation_id in elephants["extra__id"].unique():
er_io.delete_observation(observation_id)
Create new observations from existing sources
[ ]:
df_sources = er_io.get_sources()
habiba_source_id = df_sources[df_sources["manufacturer_id"] == "ST-Habiba"]["id"].values[0]
salif_keita_source_id = df_sources[df_sources["manufacturer_id"] == "ST-SalifKeita"]["id"].values[0]
habiba_source_id, salif_keita_source_id
[ ]:
observations = [
{
"fixtime": pd.Timestamp("2023-04-24 11:03:31").isoformat(),
"geometry": Point(1.99, 2.99),
"source_id": habiba_source_id, # Habiba's source
},
{
"fixtime": pd.Timestamp("2023-02-01 14:02:05").isoformat(),
"geometry": Point(1.345, 2.3303),
"source_id": habiba_source_id, # Habiba's source
},
{
"fixtime": pd.Timestamp("2023-03-21 06:03:27").isoformat(),
"geometry": Point(3.4455, -3.4554),
"source_id": salif_keita_source_id, # Salif Keita's source
},
]
gdf = gpd.GeoDataFrame.from_dict(observations)
gdf
Post observations
[ ]:
er_io.post_observations(observations=gdf, source_id_col="source_id", recorded_at_col="fixtime")
POST SubjectSource
#
[ ]:
df_subjects = er_io.get_subjects()
salif_keita_id = df_subjects[df_subjects["name"] == "Salif Keita"]["id"].values[0]
df_sources = er_io.get_sources()
salif_keita_source_id = df_sources[df_sources["manufacturer_id"] == "ST-SalifKeita"]["id"].values[0]
salif_keita_id, salif_keita_source_id
[ ]:
er_io.post_subjectsource(
salif_keita_id, # Salif Keita subject ID
source_id=salif_keita_source_id, # Salif Keita's source
lower_bound_assigned_range=pd.Timestamp.utcnow().isoformat(),
upper_bound_assigned_range=(pd.Timestamp.utcnow() + pd.Timedelta(days=9000)).isoformat(),
additional={},
)
Export to File#
Create test data#
Skip this cell if you would like to provide your own data
[ ]:
relocs = er_io.get_subjectgroup_observations(
subject_group_name="Elephants",
filter=0,
since=since,
until=until,
include_details=True,
include_inactive=True,
include_source_details=True,
include_subject_details=True,
include_subjectsource_details=True,
)
relocs
GeoPackage (.gpkg)#
(GeoPackage does not support columns with type list
so we drop
them.)
[ ]:
relocs = relocs.select_dtypes(exclude=[list])
relocs.to_file(os.path.join(output_dir, "observations.gpkg"), layer="observations")
CSV#
[ ]:
relocs.to_csv(os.path.join(output_dir, "observations.csv"))
Python Pickle (.pkl)#
[ ]:
relocs.to_pickle(os.path.join(output_dir, "observations.pkl"))
Apache Feather (.feather)#
[ ]:
relocs.to_feather(os.path.join(output_dir, "observations.feather"))