Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
from typing import Union
import numpy as np
import pandas as pd
from sklearn.neighbors import BallTree
def hopkins(data_frame: Union[np.ndarray, pd.DataFrame], sampling_size: int) -> float:
"""Assess the clusterability of a dataset. A score between 0 and 1, a score around 0.5 express
no clusterability and a score tending to 0 express a high cluster tendency.
Examples
--------
>>> from sklearn import datasets
>>> from pyclustertend import hopkins
>>> X = datasets.load_iris().data
>>> hopkins(X,150)
0.16
"""
if type(data_frame) == np.ndarray:
data_frame = pd.DataFrame(data_frame)
data_frame_sample = sample_observation_from_dataset(data_frame, sampling_size)
sample_distances_to_nearest_neighbours = get_distance_sample_to_nearest_neighbours(
data_frame, data_frame_sample
)
uniformly_selected_observations_df = simulate_df_with_same_variation(
data_frame, sampling_size
)
df_distances_to_nearest_neighbours = get_nearest_sample(
data_frame, uniformly_selected_observations_df
)
x = sum(sample_distances_to_nearest_neighbours)
y = sum(df_distances_to_nearest_neighbours)
if x + y == 0:
raise Exception("The denominator of the hopkins statistics is null")
return x / (x + y)[0]
def get_nearest_sample(df: pd.DataFrame, uniformly_selected_observations: pd.DataFrame):
tree = BallTree(df, leaf_size=2)
dist, _ = tree.query(uniformly_selected_observations, k=1)
uniformly_df_distances_to_nearest_neighbours = dist
return uniformly_df_distances_to_nearest_neighbours
def simulate_df_with_same_variation(
df: pd.DataFrame, sampling_size: int
) -> pd.DataFrame:
max_data_frame = df.max()
obs_all = []
for min, max in zip(min_data_frame, max_data_frame):
obs = np.random.uniform(min, max, sampling_size)
obs_all.append(obs)
res = pd.DataFrame(obs_all).T
return res
def get_distance_sample_to_nearest_neighbours(df: pd.DataFrame, data_frame_sample):
tree = BallTree(df, leaf_size=2)
dist, _ = tree.query(data_frame_sample, k=2)
data_frame_sample_distances_to_nearest_neighbours = dist[:, 1]
return data_frame_sample_distances_to_nearest_neighbours
def sample_observation_from_dataset(df, sampling_size: int):
if sampling_size > df.shape[0]:
raise Exception("The number of sample of sample is bigger than the shape of D")
data_frame_sample = df.sample(n=sampling_size)
return data_frame_sample