import pandas as pd pd.set_option('display.float_format', '{:.2f}'.format) # don't want scientific notation. Floats are enough csv_path = "./p2-trees-fr.csv" df = pd.read_csv(csv_path, sep=';') import sys print('Size in MB:') sys.getsizeof(df)/(10**6) df.info() # display count of columns, lines and columns names df.head(2) # display 2 first lines df.describe()# general statistics only on number types variables (int and floats) # problems: ## 0 as min value for circumference, height ## > 250000 cm as circumference, > 800000 m as height # for column in df.columns[1:]: # df[column].value_counts() # df.columns # distribution discrète for column in ['type_emplacement', 'domanialite', 'arrondissement', 'lieu', 'id_emplacement', 'libelle_francais', 'genre', 'espece', 'variete', 'hauteur_m', 'stade_developpement', 'remarquable', 'geo_point_2d_a', 'geo_point_2d_b']: print(df[column].value_counts()) print() # print(column + ':') # print(df[column].value_counts()) # print() df.columns # columns (type of data) import numpy as np # number of 'nan' for each column column_nb_of_nans = np.array([], dtype='int') for column in df.columns: where_nan = df[df[column].isna()] column_nb_of_nans = np.concatenate((column_nb_of_nans,[len(where_nan)])) column_nb_of_nans # dataframe with number of nans per column only when one nan or more where_nans = np.where(column_nb_of_nans>0) columns_with_nans_df = pd.DataFrame([column_nb_of_nans[where_nans]], columns = df.columns[where_nans]) print('Number of nan for each variable that contains minimum 1 occurrence:') columns_with_nans_df # based on the tallest tree height : https://en.wikipedia.org/wiki/Hyperion_(tree) max_height = 116 # in meters # based on the biggest tree girth (circumference) : https://www.guinnessworldrecords.com/world-records/living-tree-with-greatest-girth max_girth = 36 # in meters max_height_outlier_df = df[(df['hauteur_m'] > max_height)] max_girth_outlier_df = df[(df['circonference_cm']/100 > max_girth)] print('height outliers trees ('+ str(len(max_height_outlier_df)) + ' occurrencies):') max_height_outlier_df max_height_outlier_df.value_counts('hauteur_m') print('girth outliers trees ('+ str(len(max_girth_outlier_df)) + ' occurrencies):') max_girth_outlier_df max_girth_outlier_df.value_counts('circonference_cm') # based on the tallest tree height : https://en.wikipedia.org/wiki/Hyperion_(tree) min_height = 0 # in meters # based on the biggest tree girth (circumference) : https://www.guinnessworldrecords.com/world-records/living-tree-with-greatest-girth min_girth = 0 # in meters min_height_outlier_df = df[(df['hauteur_m'] <= min_height)] min_girth_outlier_df = df[(df['circonference_cm']/100 <= min_girth)] print('height outliers trees ('+ str(len(min_height_outlier_df)) + ' occurrencies):') min_height_outlier_df min_height_outlier_df.value_counts('hauteur_m') print('girth outliers trees ('+ str(len(max_girth_outlier_df)) + ' occurrencies):') min_girth_outlier_df min_girth_outlier_df.value_counts('circonference_cm') # def variable_univariate_analysis(df, variable_name, variable_type, variable_title_type=None): # import numpy as np # import seaborn as sns # import matplotlib.pyplot as plt # if variable_title_type == None: # variable_title_type = variable_type # # variable_type can four types # variable_types = ['qualitative_nominal','qualitative_ordinal','quantitative_discrete','quantitative_continuous'] # variable_type = str.lower(variable_type) # if variable_type not in variable_types: # print('Error : wrong variable_type') # else: # # bars but can also be pie # if variable_type=='qualitative_nominal': # value_counts = df[variable_name].value_counts(normalize=True) # max_values = 26 # if len(value_counts) >= 26: # top_values = value_counts[0:max_values] # others = value_counts[max_values:].sum() # if variable_title_type == variable_type: # others_label = 'others' # else: # others_label = '> ' + str(max_values) # top_values[others_label] = others # value_counts = top_values # value_counts.plot(kind='bar') # elif variable_type == 'qualitative_ordinal': # df.sort_values(by=variable_name, ascending=True, inplace=True) # variable_univariate_analysis(df, variable_name, 'qualitative_nominal', variable_type) # pass # elif variable_type == 'quantitative_discrete': # df.sort_values(by=variable_name, ascending=True, inplace=True) # value_counts = df[variable_name].value_counts(normalize=True) # value_counts.plot(kind='bar') # pass # elif variable_type == 'quantitative_continuous': # sns.histplot(df[variable_name][df[variable_name] <= df[variable].quantile(0.99)], stat='frequency', kde=True) # pass # if variable_type != 'qualitative_ordinal': # plt.title(variable_name + ' ( ' + variable_title_type + ' )') # plt.show() # import numpy as np # import seaborn as sns # import matplotlib.pyplot as plt # import pandas as pd # def plot_variable_distribution(df: pd.DataFrame, variable_name: str, variable_type: str, variable_title_type: str = None) -> None: # """ # Plots the distribution of a variable based on its type. # Args: # df (pd.DataFrame): The DataFrame containing the data. # variable_name (str): The name of the variable to plot. # variable_type (str): The type of the variable (e.g., 'qualitative_nominal', 'qualitative_ordinal', # 'quantitative_discrete', 'quantitative_continuous'). # variable_title_type (str, optional): An optional title type for the variable. Defaults to None. # Raises: # ValueError: If the variable_type is not recognized. # """ # if variable_title_type is None: # variable_title_type = variable_type # variable_types = ['qualitative_nominal', 'qualitative_ordinal', 'quantitative_discrete', 'quantitative_continuous'] # variable_type = variable_type.lower() # if variable_type not in variable_types: # raise ValueError('Error: wrong variable_type') # if variable_type == 'qualitative_nominal': # value_counts = df[variable_name].value_counts(normalize=True) # max_values = 26 # if len(value_counts) >= max_values: # top_values = value_counts.head(max_values) # others = value_counts.iloc[max_values:].sum() # others_label = 'others' if variable_title_type == variable_type else f'> {max_values}' # top_values[others_label] = others # value_counts = top_values # value_counts.plot(kind='bar') # elif variable_type == 'qualitative_ordinal': # df.sort_values(by=variable_name, ascending=True, inplace=True) # plot_variable_distribution(df, variable_name, 'qualitative_nominal', variable_type) # elif variable_type == 'quantitative_discrete': # df.sort_values(by=variable_name, ascending=True, inplace=True) # value_counts = df[variable_name].value_counts(normalize=True) # value_counts.plot(kind='bar') # elif variable_type == 'quantitative_continuous': # sns.histplot(df[variable_name][df[variable_name] <= df[variable_name].quantile(0.99)], stat='frequency', kde=True) # if variable_type != 'qualitative_ordinal': # plt.title(f"{variable_name} ({variable_title_type})") # plt.show() import pandas as pd import matplotlib.pyplot as plt import seaborn as sns import folium from folium.plugins import MarkerCluster import branca from IPython.display import display from scipy.spatial import cKDTree from geopy.distance import geodesic def plot_variable_distribution( df: pd.DataFrame, variable_name: str, graph_type: str, variable_title_type: str = None, ) -> None: """ Plots the distribution of a specified variable in a DataFrame based on the graph type. Args: df (pd.DataFrame): The DataFrame containing the data. variable_name (str): The name of the variable to plot. graph_type (str): The type of graph to plot (e.g., 'pie', 'heat_map', 'bars', 'hist', 'bars_log'). variable_title_type (str, optional): An optional title type for the variable. Defaults to None. Raises: ValueError: If the graph_type is not recognized or if latitude/longitude columns are missing. """ if variable_title_type is None: variable_title_type = graph_type graph_types = ['pie', 'heat_map', 'bars', 'hist', 'bars_log', 'boxplot'] graph_type = graph_type.lower() if graph_type not in graph_types: raise ValueError('Error: wrong graph_type') if graph_type == 'pie': value_counts = df[variable_name].value_counts(normalize=True) value_counts_percent = value_counts * 100 # Convert to percentage # Group classes with less than 2% into "Others" others_count = value_counts_percent[value_counts_percent < 2] if not others_count.empty: others_label = f"Others ({', '.join(others_count.index)})" value_counts = value_counts_percent[value_counts_percent >= 2] value_counts[others_label] = others_count.sum() # Add the "Others" category plt.pie( value_counts, labels=[f"{label} ({count:.1f}%)" for label, count in zip(value_counts.index, value_counts.values)], autopct='%1.1f%%' ) plt.title(f"{variable_name} Distribution") elif graph_type == 'heat_map': # Check if the DataFrame contains the necessary geographic columns if 'geo_point_2d_a' in df.columns and 'geo_point_2d_b' in df.columns and variable_name in df.columns: # Group by the variable of interest (e.g., 'city', 'region') and count the occurrences in each sector sector_data = df.groupby([variable_name]).agg({ 'geo_point_2d_a': 'mean', 'geo_point_2d_b': 'mean', }).reset_index() # Add a "population density" column based on the number of occurrences of the variable in each sector sector_data['counts'] = df.groupby(variable_name).size().values # Calculate the occurrence percentages total_counts = sector_data['counts'].sum() sector_data['percentage'] = (sector_data['counts'] / total_counts) * 100 # Common zoom level used for city-level maps in Folium (and Leaflet) zoom_level = 12 # Create a map centered around the average geographic points m = folium.Map(location=[sector_data['geo_point_2d_a'].mean(), sector_data['geo_point_2d_b'].mean()], zoom_start=zoom_level) # Normalize population densities for the heat map min_density = sector_data['counts'].min() max_density = sector_data['counts'].max() # Create the colormap with branca colormap = branca.colormap.LinearColormap( colors=['yellow', 'orange', 'red'], # Choose a classic heatmap palette (yellow-orange-red) vmin=min_density, vmax=max_density ).to_step(10) # Divide the colormap into 10 steps for the legend # Extract coordinates of the sectors coords = sector_data[['geo_point_2d_a', 'geo_point_2d_b']].values tree = cKDTree(coords) # Create the k-d tree with the coordinates # Initialize the minimum distance to a very large number min_distance = float('inf') closest_pair = None # Find the closest pairs for i, point in enumerate(coords): # Find the closest neighbor from point i (excluding the point itself) distances, indices = tree.query(point, k=2) # k=2 to get the closest neighbor and itself # The second closest will be the nearest (the first is the point itself) closest_distance = distances[1] closest_index = indices[1] # If the found distance is smaller, update the minimum distance and associated points if closest_distance < min_distance: min_distance = closest_distance closest_pair = (point, coords[closest_index]) # Convert the minimum Euclidean distance to geodesic distance for more accuracy min_distance = geodesic(closest_pair[0], closest_pair[1]).meters # Limit the circle size to half the smallest distance between two points max_circle_radius = min_distance / 2 # Average Earth radius in meters EARTH_RADIUS = 6371000 # Calculate the size of a pixel in meters (at the given average latitude) lat_rad = np.radians(np.mean(sector_data['geo_point_2d_b'])) meters_per_pixel = (2 * np.pi * EARTH_RADIUS) / (256 * (2 ** zoom_level)) * np.cos(lat_rad) max_circle_radius_in_pixels = max_circle_radius / meters_per_pixel # Add each sector as a colored circle based on population density for _, row in sector_data.iterrows(): color = colormap(row['counts']) # Use the colormap to get the color based on density # Calculate the size of the marker based on density # size = np.log(row['population_density'] + 1) * 5 # Logarithmic for a reasonable scale # size = max(10, min(size, max_circle_radius)) # Limit the size to max_circle_radius size = max_circle_radius_in_pixels # Add a circle to the map folium.CircleMarker( location=[row['geo_point_2d_a'], row['geo_point_2d_b']], radius=size, # Circle size based on density and minimum distance color=color, fill=True, fill_color=color, # Fill with the same color fill_opacity=0.7, popup=f"{variable_name}: {row[variable_name]}
Trees: {row['counts']}
Percentage: {row['percentage']:.3}%", # Allow the size to adjust based on zoom level zoom_on_click=True ).add_to(m) # Add the colormap legend to the map colormap.caption = f"Trees per {variable_name}" colormap.add_to(m) # Force the legend to the bottom left corner using CSS style m.get_root().html.add_child(folium.Element(""" """)) # Display the map display(m) m.save(f'./map_{variable_name}.html') return # End the function else: raise ValueError("The DataFrame must contain 'geo_point_2d_a', 'geo_point_2d_b', and the sector variable to display a map.") elif graph_type in ['bars', 'bars_log']: value_counts = df[variable_name].value_counts() # Group smaller classes into "others" if there are more than 26 if len(value_counts) > 26: top_categories = value_counts[:25] others_count = value_counts[25:].sum() value_counts = pd.concat([top_categories, pd.Series({'others': others_count})]) total = value_counts.sum() plt.figure(figsize=(10, 6)) ax = sns.barplot(x=value_counts.index, y=value_counts.values) plt.title(f"{variable_name} Distribution") # Add percentages inside the bars for p, percentage in zip(ax.patches, value_counts / total * 100): rotation = 90 if len(value_counts) > 10 else 0 ax.annotate( f"{percentage:.1f}%", (p.get_x() + p.get_width() * 0.50, p.get_height() * 0.90), ha='center', va='center', fontsize=10, rotation=rotation, color='black' ) plt.xlabel(variable_name) plt.ylabel('Count') if graph_type == 'bars_log': plt.yscale('log') plt.xticks(rotation=90 if len(value_counts) > 10 else 0) # Rotate labels if too many plt.show() elif graph_type == 'hist': plt.figure(figsize=(10, 6)) sns.histplot(df[variable_name][df[variable_name] <= df[variable].quantile(0.99)], kde=True) # sns.histplot(df[variable_name], kde=True, color='blue', bins=15) plt.title(f"{variable_name} Distribution") plt.xlabel(variable_name) plt.ylabel('Count') plt.show() elif graph_type == "boxplot": sns.boxplot(y=df[variable_name]) plt.title(f"{variable_name} Distribution (Boxplot)") plt.ylabel(variable_name) df.columns df['id_emplacement'].unique() df.value_counts('id_emplacement') df.value_counts('lieu') df.columns df.value_counts('variete') # interesting_variables = {'domanialite':'qualitative_nominal','arrondissement':'qualitative_nominal', # 'lieu':'qualitative_nominal', 'libelle_francais':'qualitative_nominal', # 'genre':'qualitative_nominal', 'espece':'qualitative_nominal', # 'variete':'qualitative_nominal','circonference_cm':'quantitative_continuous', # 'hauteur_m':'quantitative_continuous','stade_developpement':'qualitative_ordinal', # 'remarquable':'qualitative_ordinal'} interesting_variables = {'domanialite':'pie','arrondissement':'heat_map', 'lieu':'heat_map', 'libelle_francais':'bars', 'genre':'bars', 'espece':'bars', 'variete':'bars','circonference_cm':'hist', 'hauteur_m':'hist','stade_developpement':'pie', 'remarquable':'bars_log'} for variable in interesting_variables: plot_variable_distribution(df, variable, interesting_variables[variable]) interesting_variables variable = 'hauteur_m' # variable_univariate_analysis(df, variable, interesting_variables[variable]) import seaborn as sns import matplotlib.pyplot as plt # %matplotlib widget # ax = sns.histplot(df[variable], bins=20, kde=True) filtered_data = df[variable][df[variable]<=40] # sns.histplot(df[variable][df[variable] <= df[variable].quantile(0.99)], stat='frequency', binwidth = 1, binrange=(0,20), kde=True) sns.histplot(df[variable][df[variable] <= df[variable].quantile(0.99)], stat='frequency', kde=True) # plt.show() variable = 'domanialite' variable_univariate_analysis(df, variable, interesting_variables[variable]) interesting_variables = {'circonference_cm':'boxplot', 'hauteur_m':'boxplot'} for variable in interesting_variables: plot_variable_distribution(df, variable, interesting_variables[variable]) # genre and libelle_francais, espece etc. : search for same trees with more metadata : step 4 # regarder si les arbres avec hauteur et circonférence nulle se rencontrent ou pas. Graines plantées ? # Relevés non faits (pourquoi on donnerait une position, alors?) ## 4 : traitement valeurs aberrantes # report this tree to Paris City : the owner is unknown --> look for similar trees in order to find the right owner : step 4 df[df['domanialite'].isna()] import folium import missingno # pistes amélioration : proposer trajet pour arbres de plus de x mètres (nacelle) # regarder sur carte à quoi correspondent les id d'emplacements, par rapport aux lieux