568 lines
20 KiB
Text
568 lines
20 KiB
Text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import pandas as pd
|
|
pd.set_option('display.float_format', '{:.2f}'.format) # don't want scientific notation. Floats are enough
|
|
|
|
csv_path = "./p2-trees-fr.csv"
|
|
df = pd.read_csv(csv_path, sep=';')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import sys
|
|
print('Size in MB:')
|
|
sys.getsizeof(df)/(10**6)
|
|
|
|
|
|
df.info() # display count of columns, lines and columns names
|
|
|
|
|
|
df.head(2) # display 2 first lines
|
|
|
|
|
|
df.describe()# general statistics only on number types variables (int and floats)
|
|
|
|
# problems:
|
|
## 0 as min value for circumference, height
|
|
## > 250000 cm as circumference, > 800000 m as height
|
|
|
|
|
|
# for column in df.columns[1:]:
|
|
# df[column].value_counts()
|
|
# df.columns
|
|
# distribution discrète
|
|
for column in ['type_emplacement', 'domanialite', 'arrondissement',
|
|
'lieu', 'id_emplacement', 'libelle_francais', 'genre', 'espece', 'variete',
|
|
'hauteur_m', 'stade_developpement', 'remarquable', 'geo_point_2d_a',
|
|
'geo_point_2d_b']:
|
|
print(df[column].value_counts())
|
|
print()
|
|
# print(column + ':')
|
|
# print(df[column].value_counts())
|
|
# print()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
df.columns # columns (type of data)
|
|
|
|
|
|
import numpy as np
|
|
|
|
|
|
# number of 'nan' for each column
|
|
column_nb_of_nans = np.array([], dtype='int')
|
|
for column in df.columns:
|
|
where_nan = df[df[column].isna()]
|
|
column_nb_of_nans = np.concatenate((column_nb_of_nans,[len(where_nan)]))
|
|
column_nb_of_nans
|
|
|
|
|
|
# dataframe with number of nans per column only when one nan or more
|
|
where_nans = np.where(column_nb_of_nans>0)
|
|
columns_with_nans_df = pd.DataFrame([column_nb_of_nans[where_nans]], columns = df.columns[where_nans])
|
|
print('Number of nan for each variable that contains minimum 1 occurrence:')
|
|
columns_with_nans_df
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# based on the tallest tree height : https://en.wikipedia.org/wiki/Hyperion_(tree)
|
|
max_height = 116 # in meters
|
|
# based on the biggest tree girth (circumference) : https://www.guinnessworldrecords.com/world-records/living-tree-with-greatest-girth
|
|
max_girth = 36 # in meters
|
|
|
|
|
|
max_height_outlier_df = df[(df['hauteur_m'] > max_height)]
|
|
max_girth_outlier_df = df[(df['circonference_cm']/100 > max_girth)]
|
|
|
|
|
|
print('height outliers trees ('+ str(len(max_height_outlier_df)) + ' occurrencies):')
|
|
max_height_outlier_df
|
|
|
|
|
|
|
|
max_height_outlier_df.value_counts('hauteur_m')
|
|
|
|
|
|
print('girth outliers trees ('+ str(len(max_girth_outlier_df)) + ' occurrencies):')
|
|
max_girth_outlier_df
|
|
|
|
|
|
max_girth_outlier_df.value_counts('circonference_cm')
|
|
|
|
|
|
|
|
|
|
|
|
# based on the tallest tree height : https://en.wikipedia.org/wiki/Hyperion_(tree)
|
|
min_height = 0 # in meters
|
|
# based on the biggest tree girth (circumference) : https://www.guinnessworldrecords.com/world-records/living-tree-with-greatest-girth
|
|
min_girth = 0 # in meters
|
|
|
|
|
|
min_height_outlier_df = df[(df['hauteur_m'] <= min_height)]
|
|
min_girth_outlier_df = df[(df['circonference_cm']/100 <= min_girth)]
|
|
|
|
|
|
print('height outliers trees ('+ str(len(min_height_outlier_df)) + ' occurrencies):')
|
|
min_height_outlier_df
|
|
|
|
|
|
|
|
min_height_outlier_df.value_counts('hauteur_m')
|
|
|
|
|
|
print('girth outliers trees ('+ str(len(max_girth_outlier_df)) + ' occurrencies):')
|
|
min_girth_outlier_df
|
|
|
|
|
|
min_girth_outlier_df.value_counts('circonference_cm')
|
|
|
|
|
|
|
|
|
|
|
|
# def variable_univariate_analysis(df, variable_name, variable_type, variable_title_type=None):
|
|
# import numpy as np
|
|
# import seaborn as sns
|
|
# import matplotlib.pyplot as plt
|
|
|
|
# if variable_title_type == None:
|
|
# variable_title_type = variable_type
|
|
|
|
# # variable_type can four types
|
|
# variable_types = ['qualitative_nominal','qualitative_ordinal','quantitative_discrete','quantitative_continuous']
|
|
# variable_type = str.lower(variable_type)
|
|
# if variable_type not in variable_types:
|
|
# print('Error : wrong variable_type')
|
|
# else:
|
|
# # bars but can also be pie
|
|
# if variable_type=='qualitative_nominal':
|
|
# value_counts = df[variable_name].value_counts(normalize=True)
|
|
# max_values = 26
|
|
# if len(value_counts) >= 26:
|
|
# top_values = value_counts[0:max_values]
|
|
# others = value_counts[max_values:].sum()
|
|
# if variable_title_type == variable_type:
|
|
# others_label = 'others'
|
|
# else:
|
|
# others_label = '> ' + str(max_values)
|
|
# top_values[others_label] = others
|
|
# value_counts = top_values
|
|
# value_counts.plot(kind='bar')
|
|
# elif variable_type == 'qualitative_ordinal':
|
|
# df.sort_values(by=variable_name, ascending=True, inplace=True)
|
|
# variable_univariate_analysis(df, variable_name, 'qualitative_nominal', variable_type)
|
|
# pass
|
|
# elif variable_type == 'quantitative_discrete':
|
|
# df.sort_values(by=variable_name, ascending=True, inplace=True)
|
|
# value_counts = df[variable_name].value_counts(normalize=True)
|
|
# value_counts.plot(kind='bar')
|
|
# pass
|
|
# elif variable_type == 'quantitative_continuous':
|
|
# sns.histplot(df[variable_name][df[variable_name] <= df[variable].quantile(0.99)], stat='frequency', kde=True)
|
|
# pass
|
|
|
|
# if variable_type != 'qualitative_ordinal':
|
|
# plt.title(variable_name + ' ( ' + variable_title_type + ' )')
|
|
# plt.show()
|
|
|
|
|
|
# import numpy as np
|
|
# import seaborn as sns
|
|
# import matplotlib.pyplot as plt
|
|
# import pandas as pd
|
|
|
|
# def plot_variable_distribution(df: pd.DataFrame, variable_name: str, variable_type: str, variable_title_type: str = None) -> None:
|
|
# """
|
|
# Plots the distribution of a variable based on its type.
|
|
|
|
# Args:
|
|
# df (pd.DataFrame): The DataFrame containing the data.
|
|
# variable_name (str): The name of the variable to plot.
|
|
# variable_type (str): The type of the variable (e.g., 'qualitative_nominal', 'qualitative_ordinal',
|
|
# 'quantitative_discrete', 'quantitative_continuous').
|
|
# variable_title_type (str, optional): An optional title type for the variable. Defaults to None.
|
|
|
|
# Raises:
|
|
# ValueError: If the variable_type is not recognized.
|
|
# """
|
|
# if variable_title_type is None:
|
|
# variable_title_type = variable_type
|
|
|
|
# variable_types = ['qualitative_nominal', 'qualitative_ordinal', 'quantitative_discrete', 'quantitative_continuous']
|
|
# variable_type = variable_type.lower()
|
|
|
|
# if variable_type not in variable_types:
|
|
# raise ValueError('Error: wrong variable_type')
|
|
|
|
# if variable_type == 'qualitative_nominal':
|
|
# value_counts = df[variable_name].value_counts(normalize=True)
|
|
# max_values = 26
|
|
# if len(value_counts) >= max_values:
|
|
# top_values = value_counts.head(max_values)
|
|
# others = value_counts.iloc[max_values:].sum()
|
|
# others_label = 'others' if variable_title_type == variable_type else f'> {max_values}'
|
|
# top_values[others_label] = others
|
|
# value_counts = top_values
|
|
# value_counts.plot(kind='bar')
|
|
|
|
# elif variable_type == 'qualitative_ordinal':
|
|
# df.sort_values(by=variable_name, ascending=True, inplace=True)
|
|
# plot_variable_distribution(df, variable_name, 'qualitative_nominal', variable_type)
|
|
|
|
# elif variable_type == 'quantitative_discrete':
|
|
# df.sort_values(by=variable_name, ascending=True, inplace=True)
|
|
# value_counts = df[variable_name].value_counts(normalize=True)
|
|
# value_counts.plot(kind='bar')
|
|
|
|
# elif variable_type == 'quantitative_continuous':
|
|
# sns.histplot(df[variable_name][df[variable_name] <= df[variable_name].quantile(0.99)], stat='frequency', kde=True)
|
|
|
|
# if variable_type != 'qualitative_ordinal':
|
|
# plt.title(f"{variable_name} ({variable_title_type})")
|
|
# plt.show()
|
|
|
|
|
|
import pandas as pd
|
|
import matplotlib.pyplot as plt
|
|
import seaborn as sns
|
|
import folium
|
|
from folium.plugins import MarkerCluster
|
|
import branca
|
|
from IPython.display import display
|
|
from scipy.spatial import cKDTree
|
|
from geopy.distance import geodesic
|
|
|
|
|
|
def plot_variable_distribution(
|
|
df: pd.DataFrame,
|
|
variable_name: str,
|
|
graph_type: str,
|
|
variable_title_type: str = None,
|
|
) -> None:
|
|
"""
|
|
Plots the distribution of a specified variable in a DataFrame based on the graph type.
|
|
|
|
Args:
|
|
df (pd.DataFrame): The DataFrame containing the data.
|
|
variable_name (str): The name of the variable to plot.
|
|
graph_type (str): The type of graph to plot (e.g., 'pie', 'heat_map', 'bars', 'hist', 'bars_log').
|
|
variable_title_type (str, optional): An optional title type for the variable. Defaults to None.
|
|
|
|
Raises:
|
|
ValueError: If the graph_type is not recognized or if latitude/longitude columns are missing.
|
|
"""
|
|
if variable_title_type is None:
|
|
variable_title_type = graph_type
|
|
|
|
graph_types = ['pie', 'heat_map', 'bars', 'hist', 'bars_log', 'boxplot']
|
|
graph_type = graph_type.lower()
|
|
|
|
if graph_type not in graph_types:
|
|
raise ValueError('Error: wrong graph_type')
|
|
|
|
if graph_type == 'pie':
|
|
value_counts = df[variable_name].value_counts(normalize=True)
|
|
value_counts_percent = value_counts * 100 # Convert to percentage
|
|
|
|
# Group classes with less than 2% into "Others"
|
|
others_count = value_counts_percent[value_counts_percent < 2]
|
|
if not others_count.empty:
|
|
others_label = f"Others ({', '.join(others_count.index)})"
|
|
value_counts = value_counts_percent[value_counts_percent >= 2]
|
|
value_counts[others_label] = others_count.sum() # Add the "Others" category
|
|
|
|
plt.pie(
|
|
value_counts,
|
|
labels=[f"{label} ({count:.1f}%)" for label, count in zip(value_counts.index, value_counts.values)],
|
|
autopct='%1.1f%%'
|
|
)
|
|
plt.title(f"{variable_name} Distribution")
|
|
|
|
elif graph_type == 'heat_map':
|
|
# Check if the DataFrame contains the necessary geographic columns
|
|
if 'geo_point_2d_a' in df.columns and 'geo_point_2d_b' in df.columns and variable_name in df.columns:
|
|
# Group by the variable of interest (e.g., 'city', 'region') and count the occurrences in each sector
|
|
sector_data = df.groupby([variable_name]).agg({
|
|
'geo_point_2d_a': 'mean',
|
|
'geo_point_2d_b': 'mean',
|
|
}).reset_index()
|
|
|
|
# Add a "population density" column based on the number of occurrences of the variable in each sector
|
|
sector_data['counts'] = df.groupby(variable_name).size().values
|
|
|
|
# Calculate the occurrence percentages
|
|
total_counts = sector_data['counts'].sum()
|
|
sector_data['percentage'] = (sector_data['counts'] / total_counts) * 100
|
|
|
|
# Common zoom level used for city-level maps in Folium (and Leaflet)
|
|
zoom_level = 12
|
|
# Create a map centered around the average geographic points
|
|
m = folium.Map(location=[sector_data['geo_point_2d_a'].mean(), sector_data['geo_point_2d_b'].mean()], zoom_start=zoom_level)
|
|
|
|
# Normalize population densities for the heat map
|
|
min_density = sector_data['counts'].min()
|
|
max_density = sector_data['counts'].max()
|
|
|
|
# Create the colormap with branca
|
|
colormap = branca.colormap.LinearColormap(
|
|
colors=['yellow', 'orange', 'red'], # Choose a classic heatmap palette (yellow-orange-red)
|
|
vmin=min_density, vmax=max_density
|
|
).to_step(10) # Divide the colormap into 10 steps for the legend
|
|
|
|
# Extract coordinates of the sectors
|
|
coords = sector_data[['geo_point_2d_a', 'geo_point_2d_b']].values
|
|
tree = cKDTree(coords) # Create the k-d tree with the coordinates
|
|
|
|
# Initialize the minimum distance to a very large number
|
|
min_distance = float('inf')
|
|
closest_pair = None
|
|
|
|
# Find the closest pairs
|
|
for i, point in enumerate(coords):
|
|
# Find the closest neighbor from point i (excluding the point itself)
|
|
distances, indices = tree.query(point, k=2) # k=2 to get the closest neighbor and itself
|
|
|
|
# The second closest will be the nearest (the first is the point itself)
|
|
closest_distance = distances[1]
|
|
closest_index = indices[1]
|
|
|
|
# If the found distance is smaller, update the minimum distance and associated points
|
|
if closest_distance < min_distance:
|
|
min_distance = closest_distance
|
|
closest_pair = (point, coords[closest_index])
|
|
|
|
# Convert the minimum Euclidean distance to geodesic distance for more accuracy
|
|
min_distance = geodesic(closest_pair[0], closest_pair[1]).meters
|
|
|
|
# Limit the circle size to half the smallest distance between two points
|
|
max_circle_radius = min_distance / 2
|
|
|
|
# Average Earth radius in meters
|
|
EARTH_RADIUS = 6371000
|
|
|
|
# Calculate the size of a pixel in meters (at the given average latitude)
|
|
lat_rad = np.radians(np.mean(sector_data['geo_point_2d_b']))
|
|
meters_per_pixel = (2 * np.pi * EARTH_RADIUS) / (256 * (2 ** zoom_level)) * np.cos(lat_rad)
|
|
|
|
max_circle_radius_in_pixels = max_circle_radius / meters_per_pixel
|
|
|
|
# Add each sector as a colored circle based on population density
|
|
for _, row in sector_data.iterrows():
|
|
color = colormap(row['counts']) # Use the colormap to get the color based on density
|
|
|
|
# Calculate the size of the marker based on density
|
|
# size = np.log(row['population_density'] + 1) * 5 # Logarithmic for a reasonable scale
|
|
# size = max(10, min(size, max_circle_radius)) # Limit the size to max_circle_radius
|
|
size = max_circle_radius_in_pixels
|
|
|
|
# Add a circle to the map
|
|
folium.CircleMarker(
|
|
location=[row['geo_point_2d_a'], row['geo_point_2d_b']],
|
|
radius=size, # Circle size based on density and minimum distance
|
|
color=color,
|
|
fill=True,
|
|
fill_color=color, # Fill with the same color
|
|
fill_opacity=0.7,
|
|
popup=f"{variable_name}: {row[variable_name]}<br>Trees: {row['counts']}<br>Percentage: {row['percentage']:.3}%",
|
|
# Allow the size to adjust based on zoom level
|
|
zoom_on_click=True
|
|
).add_to(m)
|
|
|
|
# Add the colormap legend to the map
|
|
colormap.caption = f"Trees per {variable_name}"
|
|
colormap.add_to(m)
|
|
|
|
# Force the legend to the bottom left corner using CSS style
|
|
m.get_root().html.add_child(folium.Element("""
|
|
<style>
|
|
.leaflet-control-colormap {
|
|
position: absolute !important;
|
|
bottom: 10px;
|
|
left: 10px;
|
|
z-index: 1000;
|
|
}
|
|
</style>
|
|
"""))
|
|
|
|
# Display the map
|
|
display(m)
|
|
m.save(f'./map_{variable_name}.html')
|
|
return # End the function
|
|
|
|
else:
|
|
raise ValueError("The DataFrame must contain 'geo_point_2d_a', 'geo_point_2d_b', and the sector variable to display a map.")
|
|
|
|
elif graph_type in ['bars', 'bars_log']:
|
|
value_counts = df[variable_name].value_counts()
|
|
|
|
# Group smaller classes into "others" if there are more than 26
|
|
if len(value_counts) > 26:
|
|
top_categories = value_counts[:25]
|
|
others_count = value_counts[25:].sum()
|
|
value_counts = pd.concat([top_categories, pd.Series({'others': others_count})])
|
|
|
|
total = value_counts.sum()
|
|
plt.figure(figsize=(10, 6))
|
|
ax = sns.barplot(x=value_counts.index, y=value_counts.values)
|
|
plt.title(f"{variable_name} Distribution")
|
|
|
|
# Add percentages inside the bars
|
|
for p, percentage in zip(ax.patches, value_counts / total * 100):
|
|
rotation = 90 if len(value_counts) > 10 else 0
|
|
ax.annotate(
|
|
f"{percentage:.1f}%",
|
|
(p.get_x() + p.get_width() * 0.50, p.get_height() * 0.90),
|
|
ha='center', va='center', fontsize=10, rotation=rotation, color='black'
|
|
)
|
|
|
|
plt.xlabel(variable_name)
|
|
plt.ylabel('Count')
|
|
if graph_type == 'bars_log':
|
|
plt.yscale('log')
|
|
plt.xticks(rotation=90 if len(value_counts) > 10 else 0) # Rotate labels if too many
|
|
plt.show()
|
|
|
|
elif graph_type == 'hist':
|
|
plt.figure(figsize=(10, 6))
|
|
sns.histplot(df[variable_name][df[variable_name] <= df[variable].quantile(0.99)], kde=True)
|
|
# sns.histplot(df[variable_name], kde=True, color='blue', bins=15)
|
|
plt.title(f"{variable_name} Distribution")
|
|
plt.xlabel(variable_name)
|
|
plt.ylabel('Count')
|
|
plt.show()
|
|
|
|
elif graph_type == "boxplot":
|
|
sns.boxplot(y=df[variable_name])
|
|
plt.title(f"{variable_name} Distribution (Boxplot)")
|
|
plt.ylabel(variable_name)
|
|
|
|
|
|
|
|
|
|
df.columns
|
|
|
|
|
|
df['id_emplacement'].unique()
|
|
df.value_counts('id_emplacement')
|
|
|
|
|
|
df.value_counts('lieu')
|
|
|
|
|
|
df.columns
|
|
|
|
|
|
df.value_counts('variete')
|
|
|
|
|
|
# interesting_variables = {'domanialite':'qualitative_nominal','arrondissement':'qualitative_nominal',
|
|
# 'lieu':'qualitative_nominal', 'libelle_francais':'qualitative_nominal',
|
|
# 'genre':'qualitative_nominal', 'espece':'qualitative_nominal',
|
|
# 'variete':'qualitative_nominal','circonference_cm':'quantitative_continuous',
|
|
# 'hauteur_m':'quantitative_continuous','stade_developpement':'qualitative_ordinal',
|
|
# 'remarquable':'qualitative_ordinal'}
|
|
|
|
|
|
interesting_variables = {'domanialite':'pie','arrondissement':'heat_map',
|
|
'lieu':'heat_map', 'libelle_francais':'bars',
|
|
'genre':'bars', 'espece':'bars',
|
|
'variete':'bars','circonference_cm':'hist',
|
|
'hauteur_m':'hist','stade_developpement':'pie',
|
|
'remarquable':'bars_log'}
|
|
|
|
|
|
for variable in interesting_variables:
|
|
plot_variable_distribution(df, variable, interesting_variables[variable])
|
|
|
|
|
|
interesting_variables
|
|
|
|
|
|
variable = 'hauteur_m'
|
|
# variable_univariate_analysis(df, variable, interesting_variables[variable])
|
|
import seaborn as sns
|
|
import matplotlib.pyplot as plt
|
|
# %matplotlib widget
|
|
# ax = sns.histplot(df[variable], bins=20, kde=True)
|
|
filtered_data = df[variable][df[variable]<=40]
|
|
# sns.histplot(df[variable][df[variable] <= df[variable].quantile(0.99)], stat='frequency', binwidth = 1, binrange=(0,20), kde=True)
|
|
sns.histplot(df[variable][df[variable] <= df[variable].quantile(0.99)], stat='frequency', kde=True)
|
|
# plt.show()
|
|
|
|
|
|
variable = 'domanialite'
|
|
variable_univariate_analysis(df, variable, interesting_variables[variable])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
interesting_variables = {'circonference_cm':'boxplot',
|
|
'hauteur_m':'boxplot'}
|
|
|
|
|
|
for variable in interesting_variables:
|
|
plot_variable_distribution(df, variable, interesting_variables[variable])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# genre and libelle_francais, espece etc. : search for same trees with more metadata : step 4
|
|
|
|
|
|
# regarder si les arbres avec hauteur et circonférence nulle se rencontrent ou pas. Graines plantées ?
|
|
# Relevés non faits (pourquoi on donnerait une position, alors?)
|
|
|
|
|
|
## 4 : traitement valeurs aberrantes
|
|
# report this tree to Paris City : the owner is unknown --> look for similar trees in order to find the right owner : step 4
|
|
df[df['domanialite'].isna()]
|
|
|
|
|
|
import folium
|
|
import missingno
|
|
|
|
|
|
|
|
|
|
|
|
# pistes amélioration : proposer trajet pour arbres de plus de x mètres (nacelle)
|
|
# regarder sur carte à quoi correspondent les id d'emplacements, par rapport aux lieux
|
|
|
|
|
|
|