deletions
This commit is contained in:
parent
978d8601d4
commit
a6c55bf677
6 changed files with 0 additions and 354553 deletions
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,250 +0,0 @@
|
||||||
import pandas as pd
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
import seaborn as sns
|
|
||||||
import folium
|
|
||||||
from folium.plugins import MarkerCluster
|
|
||||||
import branca
|
|
||||||
from IPython.display import display
|
|
||||||
from scipy.spatial import cKDTree
|
|
||||||
from geopy.distance import geodesic
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
|
|
||||||
def plot_univariate_variable_analysis(
|
|
||||||
df: pd.DataFrame,
|
|
||||||
variable_name: str,
|
|
||||||
graph_type: str,
|
|
||||||
variable_title_type: str = None,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Plots the distribution of a specified variable in a DataFrame based on the graph type.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
df (pd.DataFrame): The DataFrame containing the data.
|
|
||||||
variable_name (str): The name of the variable to plot.
|
|
||||||
graph_type (str): The type of graph to plot (e.g., 'pie', 'heat_map', 'bars', 'hist', 'bars_log').
|
|
||||||
variable_title_type (str, optional): An optional title type for the variable. Defaults to None.
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
ValueError: If the graph_type is not recognized or if latitude/longitude columns are missing.
|
|
||||||
"""
|
|
||||||
if variable_title_type is None:
|
|
||||||
variable_title_type = graph_type
|
|
||||||
|
|
||||||
graph_types = ['pie', 'heat_map', 'bars', 'hist', 'bars_log', 'boxplot']
|
|
||||||
graph_type = graph_type.lower()
|
|
||||||
|
|
||||||
if graph_type not in graph_types:
|
|
||||||
raise ValueError('Error: wrong graph_type')
|
|
||||||
|
|
||||||
if graph_type == 'pie':
|
|
||||||
value_counts = df[variable_name].value_counts(normalize=True)
|
|
||||||
value_counts_percent = value_counts * 100 # Convert to percentage
|
|
||||||
|
|
||||||
# Group classes with less than 2% into "Others"
|
|
||||||
others_count = value_counts_percent[value_counts_percent < 2]
|
|
||||||
if not others_count.empty:
|
|
||||||
others_label = f"Others ({', '.join(others_count.index)})"
|
|
||||||
value_counts = value_counts_percent[value_counts_percent >= 2]
|
|
||||||
value_counts[others_label] = others_count.sum() # Add the "Others" category
|
|
||||||
|
|
||||||
plt.pie(
|
|
||||||
value_counts,
|
|
||||||
labels=[f"{label} ({count:.1f}%)" for label, count in zip(value_counts.index, value_counts.values)],
|
|
||||||
autopct='%1.1f%%'
|
|
||||||
)
|
|
||||||
plt.title(f"{variable_name} Distribution")
|
|
||||||
|
|
||||||
elif graph_type == 'heat_map':
|
|
||||||
# Check if the DataFrame contains the necessary geographic columns
|
|
||||||
if 'geo_point_2d_a' in df.columns and 'geo_point_2d_b' in df.columns and variable_name in df.columns:
|
|
||||||
# Group by the variable of interest (e.g., 'city', 'region') and count the occurrences in each sector
|
|
||||||
sector_data = df.groupby([variable_name]).agg({
|
|
||||||
'geo_point_2d_a': 'mean',
|
|
||||||
'geo_point_2d_b': 'mean',
|
|
||||||
}).reset_index()
|
|
||||||
|
|
||||||
# Add a "population density" column based on the number of occurrences of the variable in each sector
|
|
||||||
sector_data['counts'] = df.groupby(variable_name).size().values
|
|
||||||
|
|
||||||
# Calculate the occurrence percentages
|
|
||||||
total_counts = sector_data['counts'].sum()
|
|
||||||
sector_data['percentage'] = (sector_data['counts'] / total_counts) * 100
|
|
||||||
|
|
||||||
# Common zoom level used for city-level maps in Folium (and Leaflet)
|
|
||||||
zoom_level = 12
|
|
||||||
# Create a map centered around the average geographic points
|
|
||||||
m = folium.Map(location=[sector_data['geo_point_2d_a'].mean(), sector_data['geo_point_2d_b'].mean()], zoom_start=zoom_level)
|
|
||||||
|
|
||||||
# Normalize population densities for the heat map
|
|
||||||
min_density = sector_data['counts'].min()
|
|
||||||
max_density = sector_data['counts'].max()
|
|
||||||
|
|
||||||
# Create the colormap with branca
|
|
||||||
colormap = branca.colormap.LinearColormap(
|
|
||||||
colors=['yellow', 'orange', 'red'], # Choose a classic heatmap palette (yellow-orange-red)
|
|
||||||
vmin=min_density, vmax=max_density
|
|
||||||
).to_step(10) # Divide the colormap into 10 steps for the legend
|
|
||||||
|
|
||||||
# Extract coordinates of the sectors
|
|
||||||
coords = sector_data[['geo_point_2d_a', 'geo_point_2d_b']].values
|
|
||||||
tree = cKDTree(coords) # Create the k-d tree with the coordinates
|
|
||||||
|
|
||||||
# Initialize the minimum distance to a very large number
|
|
||||||
min_distance = float('inf')
|
|
||||||
closest_pair = None
|
|
||||||
|
|
||||||
# Find the closest pairs
|
|
||||||
for i, point in enumerate(coords):
|
|
||||||
# Find the closest neighbor from point i (excluding the point itself)
|
|
||||||
distances, indices = tree.query(point, k=2) # k=2 to get the closest neighbor and itself
|
|
||||||
|
|
||||||
# The second closest will be the nearest (the first is the point itself)
|
|
||||||
closest_distance = distances[1]
|
|
||||||
closest_index = indices[1]
|
|
||||||
|
|
||||||
# If the found distance is smaller, update the minimum distance and associated points
|
|
||||||
if closest_distance < min_distance:
|
|
||||||
min_distance = closest_distance
|
|
||||||
closest_pair = (point, coords[closest_index])
|
|
||||||
|
|
||||||
# Convert the minimum Euclidean distance to geodesic distance for more accuracy
|
|
||||||
min_distance = geodesic(closest_pair[0], closest_pair[1]).meters
|
|
||||||
|
|
||||||
# Limit the circle size to half the smallest distance between two points
|
|
||||||
max_circle_radius = min_distance / 2
|
|
||||||
|
|
||||||
# Average Earth radius in meters
|
|
||||||
EARTH_RADIUS = 6371000
|
|
||||||
|
|
||||||
# Calculate the size of a pixel in meters (at the given average latitude)
|
|
||||||
lat_rad = np.radians(np.mean(sector_data['geo_point_2d_b']))
|
|
||||||
meters_per_pixel = (2 * np.pi * EARTH_RADIUS) / (256 * (2 ** zoom_level)) * np.cos(lat_rad)
|
|
||||||
|
|
||||||
max_circle_radius_in_pixels = max_circle_radius / meters_per_pixel
|
|
||||||
|
|
||||||
# Add each sector as a colored circle based on population density
|
|
||||||
for _, row in sector_data.iterrows():
|
|
||||||
color = colormap(row['counts']) # Use the colormap to get the color based on density
|
|
||||||
|
|
||||||
# Calculate the size of the marker based on density
|
|
||||||
# size = np.log(row['population_density'] + 1) * 5 # Logarithmic for a reasonable scale
|
|
||||||
# size = max(10, min(size, max_circle_radius)) # Limit the size to max_circle_radius
|
|
||||||
size = max_circle_radius_in_pixels
|
|
||||||
|
|
||||||
# Add a circle to the map
|
|
||||||
folium.CircleMarker(
|
|
||||||
location=[row['geo_point_2d_a'], row['geo_point_2d_b']],
|
|
||||||
radius=size, # Circle size based on density and minimum distance
|
|
||||||
color=color,
|
|
||||||
fill=True,
|
|
||||||
fill_color=color, # Fill with the same color
|
|
||||||
fill_opacity=0.7,
|
|
||||||
popup=f"{variable_name}: {row[variable_name]}<br>Trees: {row['counts']}<br>Percentage: {row['percentage']:.3}%",
|
|
||||||
# Allow the size to adjust based on zoom level
|
|
||||||
zoom_on_click=True
|
|
||||||
).add_to(m)
|
|
||||||
|
|
||||||
# Add the colormap legend to the map
|
|
||||||
colormap.caption = f"Trees per {variable_name}"
|
|
||||||
colormap.add_to(m)
|
|
||||||
|
|
||||||
# Force the legend to the bottom left corner using CSS style
|
|
||||||
m.get_root().html.add_child(folium.Element("""
|
|
||||||
<style>
|
|
||||||
.leaflet-control-colormap {
|
|
||||||
position: absolute !important;
|
|
||||||
bottom: 10px;
|
|
||||||
left: 10px;
|
|
||||||
z-index: 1000;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
"""))
|
|
||||||
|
|
||||||
# Display the map
|
|
||||||
display(m)
|
|
||||||
m.save(f'./map_{variable_name}.html')
|
|
||||||
return # End the function
|
|
||||||
|
|
||||||
else:
|
|
||||||
raise ValueError("The DataFrame must contain 'geo_point_2d_a', 'geo_point_2d_b', and the sector variable to display a map.")
|
|
||||||
|
|
||||||
elif graph_type in ['bars', 'bars_log']:
|
|
||||||
value_counts = df[variable_name].value_counts()
|
|
||||||
|
|
||||||
# Group smaller classes into "others" if there are more than 26
|
|
||||||
if len(value_counts) > 26:
|
|
||||||
top_categories = value_counts[:25]
|
|
||||||
others_count = value_counts[25:].sum()
|
|
||||||
value_counts = pd.concat([top_categories, pd.Series({'others': others_count})])
|
|
||||||
|
|
||||||
total = value_counts.sum()
|
|
||||||
plt.figure(figsize=(10, 6))
|
|
||||||
ax = sns.barplot(x=value_counts.index, y=value_counts.values)
|
|
||||||
plt.title(f"{variable_name} Distribution")
|
|
||||||
|
|
||||||
# Add percentages inside the bars
|
|
||||||
for p, percentage in zip(ax.patches, value_counts / total * 100):
|
|
||||||
rotation = 90 if len(value_counts) > 10 else 0
|
|
||||||
ax.annotate(
|
|
||||||
f"{percentage:.1f}%",
|
|
||||||
(p.get_x() + p.get_width() * 0.50, p.get_height() * 0.90),
|
|
||||||
ha='center', va='center', fontsize=10, rotation=rotation, color='black'
|
|
||||||
)
|
|
||||||
|
|
||||||
plt.xlabel(variable_name)
|
|
||||||
plt.ylabel('Count')
|
|
||||||
if graph_type == 'bars_log':
|
|
||||||
plt.yscale('log')
|
|
||||||
plt.xticks(rotation=90 if len(value_counts) > 10 else 0) # Rotate labels if too many
|
|
||||||
plt.show()
|
|
||||||
|
|
||||||
elif graph_type == 'hist':
|
|
||||||
plt.figure(figsize=(10, 6))
|
|
||||||
# sns.histplot(df[variable_name][df[variable_name] <= df[variable].quantile(0.99)], kde=True)
|
|
||||||
sns.histplot(df[variable_name], kde=True, color='blue', bins=15)
|
|
||||||
plt.title(f"{variable_name} Distribution")
|
|
||||||
plt.xlabel(variable_name)
|
|
||||||
plt.ylabel('Count')
|
|
||||||
plt.show()
|
|
||||||
|
|
||||||
elif graph_type == "boxplot":
|
|
||||||
# sns.boxplot(y=df[variable_name][df[variable_name] <= df[variable_name].quantile(0.99)])
|
|
||||||
sns.boxplot(y=df[variable_name])
|
|
||||||
plt.title(f"{variable_name} Distribution (Boxplot)")
|
|
||||||
plt.ylabel(variable_name)
|
|
||||||
plt.show()
|
|
||||||
|
|
||||||
def compute_distances(df, tree_index):
|
|
||||||
"""
|
|
||||||
Computes distances between a specific tree and all other trees in the DataFrame.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
df: DataFrame containing tree data with 'geo_point_2d_a' (longitude) and 'geo_point_2d_b' (latitude) columns.
|
|
||||||
tree_index: Index of the specific tree in the DataFrame.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
A pandas Series containing distances to all other trees from the specified tree.
|
|
||||||
Returns None if the tree_index is out of range or if necessary columns are missing.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Safety checks
|
|
||||||
if not all(col in df.columns for col in ['geo_point_2d_a', 'geo_point_2d_b']):
|
|
||||||
print("Error: 'geo_point_2d_a' or 'geo_point_2d_b' columns not found in DataFrame.")
|
|
||||||
return None
|
|
||||||
|
|
||||||
if tree_index not in df.index:
|
|
||||||
print(f"Error: tree_index {tree_index} is out of range.")
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Get coordinates of the chosen tree
|
|
||||||
tree_coords = (df.loc[tree_index, 'geo_point_2d_a'], df.loc[tree_index, 'geo_point_2d_b'])
|
|
||||||
|
|
||||||
# Calculate distances to all other trees
|
|
||||||
distances = []
|
|
||||||
for index, row in df.iterrows():
|
|
||||||
other_tree_coords = (row['geo_point_2d_a'], row['geo_point_2d_b']) # Assuming latitude, longitude order
|
|
||||||
distance = geodesic(tree_coords, other_tree_coords).meters
|
|
||||||
distances.append(distance)
|
|
||||||
|
|
||||||
return pd.Series(distances, index=df.index) # Use original index
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,568 +0,0 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
import pandas as pd
|
|
||||||
pd.set_option('display.float_format', '{:.2f}'.format) # don't want scientific notation. Floats are enough
|
|
||||||
|
|
||||||
csv_path = "./p2-trees-fr.csv"
|
|
||||||
df = pd.read_csv(csv_path, sep=';')
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
import sys
|
|
||||||
print('Size in MB:')
|
|
||||||
sys.getsizeof(df)/(10**6)
|
|
||||||
|
|
||||||
|
|
||||||
df.info() # display count of columns, lines and columns names
|
|
||||||
|
|
||||||
|
|
||||||
df.head(2) # display 2 first lines
|
|
||||||
|
|
||||||
|
|
||||||
df.describe()# general statistics only on number types variables (int and floats)
|
|
||||||
|
|
||||||
# problems:
|
|
||||||
## 0 as min value for circumference, height
|
|
||||||
## > 250000 cm as circumference, > 800000 m as height
|
|
||||||
|
|
||||||
|
|
||||||
# for column in df.columns[1:]:
|
|
||||||
# df[column].value_counts()
|
|
||||||
# df.columns
|
|
||||||
# distribution discrète
|
|
||||||
for column in ['type_emplacement', 'domanialite', 'arrondissement',
|
|
||||||
'lieu', 'id_emplacement', 'libelle_francais', 'genre', 'espece', 'variete',
|
|
||||||
'hauteur_m', 'stade_developpement', 'remarquable', 'geo_point_2d_a',
|
|
||||||
'geo_point_2d_b']:
|
|
||||||
print(df[column].value_counts())
|
|
||||||
print()
|
|
||||||
# print(column + ':')
|
|
||||||
# print(df[column].value_counts())
|
|
||||||
# print()
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
df.columns # columns (type of data)
|
|
||||||
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
|
|
||||||
# number of 'nan' for each column
|
|
||||||
column_nb_of_nans = np.array([], dtype='int')
|
|
||||||
for column in df.columns:
|
|
||||||
where_nan = df[df[column].isna()]
|
|
||||||
column_nb_of_nans = np.concatenate((column_nb_of_nans,[len(where_nan)]))
|
|
||||||
column_nb_of_nans
|
|
||||||
|
|
||||||
|
|
||||||
# dataframe with number of nans per column only when one nan or more
|
|
||||||
where_nans = np.where(column_nb_of_nans>0)
|
|
||||||
columns_with_nans_df = pd.DataFrame([column_nb_of_nans[where_nans]], columns = df.columns[where_nans])
|
|
||||||
print('Number of nan for each variable that contains minimum 1 occurrence:')
|
|
||||||
columns_with_nans_df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# based on the tallest tree height : https://en.wikipedia.org/wiki/Hyperion_(tree)
|
|
||||||
max_height = 116 # in meters
|
|
||||||
# based on the biggest tree girth (circumference) : https://www.guinnessworldrecords.com/world-records/living-tree-with-greatest-girth
|
|
||||||
max_girth = 36 # in meters
|
|
||||||
|
|
||||||
|
|
||||||
max_height_outlier_df = df[(df['hauteur_m'] > max_height)]
|
|
||||||
max_girth_outlier_df = df[(df['circonference_cm']/100 > max_girth)]
|
|
||||||
|
|
||||||
|
|
||||||
print('height outliers trees ('+ str(len(max_height_outlier_df)) + ' occurrencies):')
|
|
||||||
max_height_outlier_df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
max_height_outlier_df.value_counts('hauteur_m')
|
|
||||||
|
|
||||||
|
|
||||||
print('girth outliers trees ('+ str(len(max_girth_outlier_df)) + ' occurrencies):')
|
|
||||||
max_girth_outlier_df
|
|
||||||
|
|
||||||
|
|
||||||
max_girth_outlier_df.value_counts('circonference_cm')
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# based on the tallest tree height : https://en.wikipedia.org/wiki/Hyperion_(tree)
|
|
||||||
min_height = 0 # in meters
|
|
||||||
# based on the biggest tree girth (circumference) : https://www.guinnessworldrecords.com/world-records/living-tree-with-greatest-girth
|
|
||||||
min_girth = 0 # in meters
|
|
||||||
|
|
||||||
|
|
||||||
min_height_outlier_df = df[(df['hauteur_m'] <= min_height)]
|
|
||||||
min_girth_outlier_df = df[(df['circonference_cm']/100 <= min_girth)]
|
|
||||||
|
|
||||||
|
|
||||||
print('height outliers trees ('+ str(len(min_height_outlier_df)) + ' occurrencies):')
|
|
||||||
min_height_outlier_df
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
min_height_outlier_df.value_counts('hauteur_m')
|
|
||||||
|
|
||||||
|
|
||||||
print('girth outliers trees ('+ str(len(max_girth_outlier_df)) + ' occurrencies):')
|
|
||||||
min_girth_outlier_df
|
|
||||||
|
|
||||||
|
|
||||||
min_girth_outlier_df.value_counts('circonference_cm')
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# def variable_univariate_analysis(df, variable_name, variable_type, variable_title_type=None):
|
|
||||||
# import numpy as np
|
|
||||||
# import seaborn as sns
|
|
||||||
# import matplotlib.pyplot as plt
|
|
||||||
|
|
||||||
# if variable_title_type == None:
|
|
||||||
# variable_title_type = variable_type
|
|
||||||
|
|
||||||
# # variable_type can four types
|
|
||||||
# variable_types = ['qualitative_nominal','qualitative_ordinal','quantitative_discrete','quantitative_continuous']
|
|
||||||
# variable_type = str.lower(variable_type)
|
|
||||||
# if variable_type not in variable_types:
|
|
||||||
# print('Error : wrong variable_type')
|
|
||||||
# else:
|
|
||||||
# # bars but can also be pie
|
|
||||||
# if variable_type=='qualitative_nominal':
|
|
||||||
# value_counts = df[variable_name].value_counts(normalize=True)
|
|
||||||
# max_values = 26
|
|
||||||
# if len(value_counts) >= 26:
|
|
||||||
# top_values = value_counts[0:max_values]
|
|
||||||
# others = value_counts[max_values:].sum()
|
|
||||||
# if variable_title_type == variable_type:
|
|
||||||
# others_label = 'others'
|
|
||||||
# else:
|
|
||||||
# others_label = '> ' + str(max_values)
|
|
||||||
# top_values[others_label] = others
|
|
||||||
# value_counts = top_values
|
|
||||||
# value_counts.plot(kind='bar')
|
|
||||||
# elif variable_type == 'qualitative_ordinal':
|
|
||||||
# df.sort_values(by=variable_name, ascending=True, inplace=True)
|
|
||||||
# variable_univariate_analysis(df, variable_name, 'qualitative_nominal', variable_type)
|
|
||||||
# pass
|
|
||||||
# elif variable_type == 'quantitative_discrete':
|
|
||||||
# df.sort_values(by=variable_name, ascending=True, inplace=True)
|
|
||||||
# value_counts = df[variable_name].value_counts(normalize=True)
|
|
||||||
# value_counts.plot(kind='bar')
|
|
||||||
# pass
|
|
||||||
# elif variable_type == 'quantitative_continuous':
|
|
||||||
# sns.histplot(df[variable_name][df[variable_name] <= df[variable].quantile(0.99)], stat='frequency', kde=True)
|
|
||||||
# pass
|
|
||||||
|
|
||||||
# if variable_type != 'qualitative_ordinal':
|
|
||||||
# plt.title(variable_name + ' ( ' + variable_title_type + ' )')
|
|
||||||
# plt.show()
|
|
||||||
|
|
||||||
|
|
||||||
# import numpy as np
|
|
||||||
# import seaborn as sns
|
|
||||||
# import matplotlib.pyplot as plt
|
|
||||||
# import pandas as pd
|
|
||||||
|
|
||||||
# def plot_variable_distribution(df: pd.DataFrame, variable_name: str, variable_type: str, variable_title_type: str = None) -> None:
|
|
||||||
# """
|
|
||||||
# Plots the distribution of a variable based on its type.
|
|
||||||
|
|
||||||
# Args:
|
|
||||||
# df (pd.DataFrame): The DataFrame containing the data.
|
|
||||||
# variable_name (str): The name of the variable to plot.
|
|
||||||
# variable_type (str): The type of the variable (e.g., 'qualitative_nominal', 'qualitative_ordinal',
|
|
||||||
# 'quantitative_discrete', 'quantitative_continuous').
|
|
||||||
# variable_title_type (str, optional): An optional title type for the variable. Defaults to None.
|
|
||||||
|
|
||||||
# Raises:
|
|
||||||
# ValueError: If the variable_type is not recognized.
|
|
||||||
# """
|
|
||||||
# if variable_title_type is None:
|
|
||||||
# variable_title_type = variable_type
|
|
||||||
|
|
||||||
# variable_types = ['qualitative_nominal', 'qualitative_ordinal', 'quantitative_discrete', 'quantitative_continuous']
|
|
||||||
# variable_type = variable_type.lower()
|
|
||||||
|
|
||||||
# if variable_type not in variable_types:
|
|
||||||
# raise ValueError('Error: wrong variable_type')
|
|
||||||
|
|
||||||
# if variable_type == 'qualitative_nominal':
|
|
||||||
# value_counts = df[variable_name].value_counts(normalize=True)
|
|
||||||
# max_values = 26
|
|
||||||
# if len(value_counts) >= max_values:
|
|
||||||
# top_values = value_counts.head(max_values)
|
|
||||||
# others = value_counts.iloc[max_values:].sum()
|
|
||||||
# others_label = 'others' if variable_title_type == variable_type else f'> {max_values}'
|
|
||||||
# top_values[others_label] = others
|
|
||||||
# value_counts = top_values
|
|
||||||
# value_counts.plot(kind='bar')
|
|
||||||
|
|
||||||
# elif variable_type == 'qualitative_ordinal':
|
|
||||||
# df.sort_values(by=variable_name, ascending=True, inplace=True)
|
|
||||||
# plot_variable_distribution(df, variable_name, 'qualitative_nominal', variable_type)
|
|
||||||
|
|
||||||
# elif variable_type == 'quantitative_discrete':
|
|
||||||
# df.sort_values(by=variable_name, ascending=True, inplace=True)
|
|
||||||
# value_counts = df[variable_name].value_counts(normalize=True)
|
|
||||||
# value_counts.plot(kind='bar')
|
|
||||||
|
|
||||||
# elif variable_type == 'quantitative_continuous':
|
|
||||||
# sns.histplot(df[variable_name][df[variable_name] <= df[variable_name].quantile(0.99)], stat='frequency', kde=True)
|
|
||||||
|
|
||||||
# if variable_type != 'qualitative_ordinal':
|
|
||||||
# plt.title(f"{variable_name} ({variable_title_type})")
|
|
||||||
# plt.show()
|
|
||||||
|
|
||||||
|
|
||||||
import pandas as pd
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
import seaborn as sns
|
|
||||||
import folium
|
|
||||||
from folium.plugins import MarkerCluster
|
|
||||||
import branca
|
|
||||||
from IPython.display import display
|
|
||||||
from scipy.spatial import cKDTree
|
|
||||||
from geopy.distance import geodesic
|
|
||||||
|
|
||||||
|
|
||||||
def plot_variable_distribution(
|
|
||||||
df: pd.DataFrame,
|
|
||||||
variable_name: str,
|
|
||||||
graph_type: str,
|
|
||||||
variable_title_type: str = None,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Plots the distribution of a specified variable in a DataFrame based on the graph type.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
df (pd.DataFrame): The DataFrame containing the data.
|
|
||||||
variable_name (str): The name of the variable to plot.
|
|
||||||
graph_type (str): The type of graph to plot (e.g., 'pie', 'heat_map', 'bars', 'hist', 'bars_log').
|
|
||||||
variable_title_type (str, optional): An optional title type for the variable. Defaults to None.
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
ValueError: If the graph_type is not recognized or if latitude/longitude columns are missing.
|
|
||||||
"""
|
|
||||||
if variable_title_type is None:
|
|
||||||
variable_title_type = graph_type
|
|
||||||
|
|
||||||
graph_types = ['pie', 'heat_map', 'bars', 'hist', 'bars_log', 'boxplot']
|
|
||||||
graph_type = graph_type.lower()
|
|
||||||
|
|
||||||
if graph_type not in graph_types:
|
|
||||||
raise ValueError('Error: wrong graph_type')
|
|
||||||
|
|
||||||
if graph_type == 'pie':
|
|
||||||
value_counts = df[variable_name].value_counts(normalize=True)
|
|
||||||
value_counts_percent = value_counts * 100 # Convert to percentage
|
|
||||||
|
|
||||||
# Group classes with less than 2% into "Others"
|
|
||||||
others_count = value_counts_percent[value_counts_percent < 2]
|
|
||||||
if not others_count.empty:
|
|
||||||
others_label = f"Others ({', '.join(others_count.index)})"
|
|
||||||
value_counts = value_counts_percent[value_counts_percent >= 2]
|
|
||||||
value_counts[others_label] = others_count.sum() # Add the "Others" category
|
|
||||||
|
|
||||||
plt.pie(
|
|
||||||
value_counts,
|
|
||||||
labels=[f"{label} ({count:.1f}%)" for label, count in zip(value_counts.index, value_counts.values)],
|
|
||||||
autopct='%1.1f%%'
|
|
||||||
)
|
|
||||||
plt.title(f"{variable_name} Distribution")
|
|
||||||
|
|
||||||
elif graph_type == 'heat_map':
|
|
||||||
# Check if the DataFrame contains the necessary geographic columns
|
|
||||||
if 'geo_point_2d_a' in df.columns and 'geo_point_2d_b' in df.columns and variable_name in df.columns:
|
|
||||||
# Group by the variable of interest (e.g., 'city', 'region') and count the occurrences in each sector
|
|
||||||
sector_data = df.groupby([variable_name]).agg({
|
|
||||||
'geo_point_2d_a': 'mean',
|
|
||||||
'geo_point_2d_b': 'mean',
|
|
||||||
}).reset_index()
|
|
||||||
|
|
||||||
# Add a "population density" column based on the number of occurrences of the variable in each sector
|
|
||||||
sector_data['counts'] = df.groupby(variable_name).size().values
|
|
||||||
|
|
||||||
# Calculate the occurrence percentages
|
|
||||||
total_counts = sector_data['counts'].sum()
|
|
||||||
sector_data['percentage'] = (sector_data['counts'] / total_counts) * 100
|
|
||||||
|
|
||||||
# Common zoom level used for city-level maps in Folium (and Leaflet)
|
|
||||||
zoom_level = 12
|
|
||||||
# Create a map centered around the average geographic points
|
|
||||||
m = folium.Map(location=[sector_data['geo_point_2d_a'].mean(), sector_data['geo_point_2d_b'].mean()], zoom_start=zoom_level)
|
|
||||||
|
|
||||||
# Normalize population densities for the heat map
|
|
||||||
min_density = sector_data['counts'].min()
|
|
||||||
max_density = sector_data['counts'].max()
|
|
||||||
|
|
||||||
# Create the colormap with branca
|
|
||||||
colormap = branca.colormap.LinearColormap(
|
|
||||||
colors=['yellow', 'orange', 'red'], # Choose a classic heatmap palette (yellow-orange-red)
|
|
||||||
vmin=min_density, vmax=max_density
|
|
||||||
).to_step(10) # Divide the colormap into 10 steps for the legend
|
|
||||||
|
|
||||||
# Extract coordinates of the sectors
|
|
||||||
coords = sector_data[['geo_point_2d_a', 'geo_point_2d_b']].values
|
|
||||||
tree = cKDTree(coords) # Create the k-d tree with the coordinates
|
|
||||||
|
|
||||||
# Initialize the minimum distance to a very large number
|
|
||||||
min_distance = float('inf')
|
|
||||||
closest_pair = None
|
|
||||||
|
|
||||||
# Find the closest pairs
|
|
||||||
for i, point in enumerate(coords):
|
|
||||||
# Find the closest neighbor from point i (excluding the point itself)
|
|
||||||
distances, indices = tree.query(point, k=2) # k=2 to get the closest neighbor and itself
|
|
||||||
|
|
||||||
# The second closest will be the nearest (the first is the point itself)
|
|
||||||
closest_distance = distances[1]
|
|
||||||
closest_index = indices[1]
|
|
||||||
|
|
||||||
# If the found distance is smaller, update the minimum distance and associated points
|
|
||||||
if closest_distance < min_distance:
|
|
||||||
min_distance = closest_distance
|
|
||||||
closest_pair = (point, coords[closest_index])
|
|
||||||
|
|
||||||
# Convert the minimum Euclidean distance to geodesic distance for more accuracy
|
|
||||||
min_distance = geodesic(closest_pair[0], closest_pair[1]).meters
|
|
||||||
|
|
||||||
# Limit the circle size to half the smallest distance between two points
|
|
||||||
max_circle_radius = min_distance / 2
|
|
||||||
|
|
||||||
# Average Earth radius in meters
|
|
||||||
EARTH_RADIUS = 6371000
|
|
||||||
|
|
||||||
# Calculate the size of a pixel in meters (at the given average latitude)
|
|
||||||
lat_rad = np.radians(np.mean(sector_data['geo_point_2d_b']))
|
|
||||||
meters_per_pixel = (2 * np.pi * EARTH_RADIUS) / (256 * (2 ** zoom_level)) * np.cos(lat_rad)
|
|
||||||
|
|
||||||
max_circle_radius_in_pixels = max_circle_radius / meters_per_pixel
|
|
||||||
|
|
||||||
# Add each sector as a colored circle based on population density
|
|
||||||
for _, row in sector_data.iterrows():
|
|
||||||
color = colormap(row['counts']) # Use the colormap to get the color based on density
|
|
||||||
|
|
||||||
# Calculate the size of the marker based on density
|
|
||||||
# size = np.log(row['population_density'] + 1) * 5 # Logarithmic for a reasonable scale
|
|
||||||
# size = max(10, min(size, max_circle_radius)) # Limit the size to max_circle_radius
|
|
||||||
size = max_circle_radius_in_pixels
|
|
||||||
|
|
||||||
# Add a circle to the map
|
|
||||||
folium.CircleMarker(
|
|
||||||
location=[row['geo_point_2d_a'], row['geo_point_2d_b']],
|
|
||||||
radius=size, # Circle size based on density and minimum distance
|
|
||||||
color=color,
|
|
||||||
fill=True,
|
|
||||||
fill_color=color, # Fill with the same color
|
|
||||||
fill_opacity=0.7,
|
|
||||||
popup=f"{variable_name}: {row[variable_name]}<br>Trees: {row['counts']}<br>Percentage: {row['percentage']:.3}%",
|
|
||||||
# Allow the size to adjust based on zoom level
|
|
||||||
zoom_on_click=True
|
|
||||||
).add_to(m)
|
|
||||||
|
|
||||||
# Add the colormap legend to the map
|
|
||||||
colormap.caption = f"Trees per {variable_name}"
|
|
||||||
colormap.add_to(m)
|
|
||||||
|
|
||||||
# Force the legend to the bottom left corner using CSS style
|
|
||||||
m.get_root().html.add_child(folium.Element("""
|
|
||||||
<style>
|
|
||||||
.leaflet-control-colormap {
|
|
||||||
position: absolute !important;
|
|
||||||
bottom: 10px;
|
|
||||||
left: 10px;
|
|
||||||
z-index: 1000;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
"""))
|
|
||||||
|
|
||||||
# Display the map
|
|
||||||
display(m)
|
|
||||||
m.save(f'./map_{variable_name}.html')
|
|
||||||
return # End the function
|
|
||||||
|
|
||||||
else:
|
|
||||||
raise ValueError("The DataFrame must contain 'geo_point_2d_a', 'geo_point_2d_b', and the sector variable to display a map.")
|
|
||||||
|
|
||||||
elif graph_type in ['bars', 'bars_log']:
|
|
||||||
value_counts = df[variable_name].value_counts()
|
|
||||||
|
|
||||||
# Group smaller classes into "others" if there are more than 26
|
|
||||||
if len(value_counts) > 26:
|
|
||||||
top_categories = value_counts[:25]
|
|
||||||
others_count = value_counts[25:].sum()
|
|
||||||
value_counts = pd.concat([top_categories, pd.Series({'others': others_count})])
|
|
||||||
|
|
||||||
total = value_counts.sum()
|
|
||||||
plt.figure(figsize=(10, 6))
|
|
||||||
ax = sns.barplot(x=value_counts.index, y=value_counts.values)
|
|
||||||
plt.title(f"{variable_name} Distribution")
|
|
||||||
|
|
||||||
# Add percentages inside the bars
|
|
||||||
for p, percentage in zip(ax.patches, value_counts / total * 100):
|
|
||||||
rotation = 90 if len(value_counts) > 10 else 0
|
|
||||||
ax.annotate(
|
|
||||||
f"{percentage:.1f}%",
|
|
||||||
(p.get_x() + p.get_width() * 0.50, p.get_height() * 0.90),
|
|
||||||
ha='center', va='center', fontsize=10, rotation=rotation, color='black'
|
|
||||||
)
|
|
||||||
|
|
||||||
plt.xlabel(variable_name)
|
|
||||||
plt.ylabel('Count')
|
|
||||||
if graph_type == 'bars_log':
|
|
||||||
plt.yscale('log')
|
|
||||||
plt.xticks(rotation=90 if len(value_counts) > 10 else 0) # Rotate labels if too many
|
|
||||||
plt.show()
|
|
||||||
|
|
||||||
elif graph_type == 'hist':
|
|
||||||
plt.figure(figsize=(10, 6))
|
|
||||||
sns.histplot(df[variable_name][df[variable_name] <= df[variable].quantile(0.99)], kde=True)
|
|
||||||
# sns.histplot(df[variable_name], kde=True, color='blue', bins=15)
|
|
||||||
plt.title(f"{variable_name} Distribution")
|
|
||||||
plt.xlabel(variable_name)
|
|
||||||
plt.ylabel('Count')
|
|
||||||
plt.show()
|
|
||||||
|
|
||||||
elif graph_type == "boxplot":
|
|
||||||
sns.boxplot(y=df[variable_name])
|
|
||||||
plt.title(f"{variable_name} Distribution (Boxplot)")
|
|
||||||
plt.ylabel(variable_name)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
df.columns
|
|
||||||
|
|
||||||
|
|
||||||
df['id_emplacement'].unique()
|
|
||||||
df.value_counts('id_emplacement')
|
|
||||||
|
|
||||||
|
|
||||||
df.value_counts('lieu')
|
|
||||||
|
|
||||||
|
|
||||||
df.columns
|
|
||||||
|
|
||||||
|
|
||||||
df.value_counts('variete')
|
|
||||||
|
|
||||||
|
|
||||||
# interesting_variables = {'domanialite':'qualitative_nominal','arrondissement':'qualitative_nominal',
|
|
||||||
# 'lieu':'qualitative_nominal', 'libelle_francais':'qualitative_nominal',
|
|
||||||
# 'genre':'qualitative_nominal', 'espece':'qualitative_nominal',
|
|
||||||
# 'variete':'qualitative_nominal','circonference_cm':'quantitative_continuous',
|
|
||||||
# 'hauteur_m':'quantitative_continuous','stade_developpement':'qualitative_ordinal',
|
|
||||||
# 'remarquable':'qualitative_ordinal'}
|
|
||||||
|
|
||||||
|
|
||||||
interesting_variables = {'domanialite':'pie','arrondissement':'heat_map',
|
|
||||||
'lieu':'heat_map', 'libelle_francais':'bars',
|
|
||||||
'genre':'bars', 'espece':'bars',
|
|
||||||
'variete':'bars','circonference_cm':'hist',
|
|
||||||
'hauteur_m':'hist','stade_developpement':'pie',
|
|
||||||
'remarquable':'bars_log'}
|
|
||||||
|
|
||||||
|
|
||||||
for variable in interesting_variables:
|
|
||||||
plot_variable_distribution(df, variable, interesting_variables[variable])
|
|
||||||
|
|
||||||
|
|
||||||
interesting_variables
|
|
||||||
|
|
||||||
|
|
||||||
variable = 'hauteur_m'
|
|
||||||
# variable_univariate_analysis(df, variable, interesting_variables[variable])
|
|
||||||
import seaborn as sns
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
# %matplotlib widget
|
|
||||||
# ax = sns.histplot(df[variable], bins=20, kde=True)
|
|
||||||
filtered_data = df[variable][df[variable]<=40]
|
|
||||||
# sns.histplot(df[variable][df[variable] <= df[variable].quantile(0.99)], stat='frequency', binwidth = 1, binrange=(0,20), kde=True)
|
|
||||||
sns.histplot(df[variable][df[variable] <= df[variable].quantile(0.99)], stat='frequency', kde=True)
|
|
||||||
# plt.show()
|
|
||||||
|
|
||||||
|
|
||||||
variable = 'domanialite'
|
|
||||||
variable_univariate_analysis(df, variable, interesting_variables[variable])
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
interesting_variables = {'circonference_cm':'boxplot',
|
|
||||||
'hauteur_m':'boxplot'}
|
|
||||||
|
|
||||||
|
|
||||||
for variable in interesting_variables:
|
|
||||||
plot_variable_distribution(df, variable, interesting_variables[variable])
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# genre and libelle_francais, espece etc. : search for same trees with more metadata : step 4
|
|
||||||
|
|
||||||
|
|
||||||
# regarder si les arbres avec hauteur et circonférence nulle se rencontrent ou pas. Graines plantées ?
|
|
||||||
# Relevés non faits (pourquoi on donnerait une position, alors?)
|
|
||||||
|
|
||||||
|
|
||||||
## 4 : traitement valeurs aberrantes
|
|
||||||
# report this tree to Paris City : the owner is unknown --> look for similar trees in order to find the right owner : step 4
|
|
||||||
df[df['domanialite'].isna()]
|
|
||||||
|
|
||||||
|
|
||||||
import folium
|
|
||||||
import missingno
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# pistes amélioration : proposer trajet pour arbres de plus de x mètres (nacelle)
|
|
||||||
# regarder sur carte à quoi correspondent les id d'emplacements, par rapport aux lieux
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Loading…
Add table
Reference in a new issue