Exploring Dataset

Run the following code before drawing figures. Here is the code for imports, data loading and preprocessing.

import json
import pandas as pd
import plotly.express as px
import plotly.subplots
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import pairwise
import plotly.graph_objects as go
from dash import Dash, dcc, html, Input, Output, callback, dash_table
from jinja2 import Template
from utils import getLocation, removeTextColumn

pd.DataFrame.iteritems = pd.DataFrame.items  # since iteritems is removed from pandas 2.0

quantity_model = pd.read_excel("../clean_quantity_model.xlsx", header=0)
herbs = list(quantity_model.columns[1:])
chinese_quantity_model = pd.read_excel("../chinese_quantity_model.xlsx", header=0)
herb_types = pd.read_excel("../herbs_name_animalplantseparation.xlsx", header=0, index_col=0, usecols=[1, 3])
formulae_locations = getLocation()
quantity_model = pd.merge(quantity_model, formulae_locations, how='left', on='Name')
herbs_frequency_df = pd.read_excel('../herbs_name_frequency.xlsx', header=0, index_col=0)
merge_result = pd.read_excel("../shanghanlun_mergeOCR_afterExpansion.xlsx", header=0)

# relocate the location columns
locations = quantity_model.pop('Location')
quantity_model.insert(1, locations.name, locations)

Here are the utility functions.

import pandas as pd
import plotly.graph_objects as go

herb_types = pd.read_excel("../herbs_name_animalplantseparation.xlsx", header=0, index_col=0, usecols=[1, 3])


def removeTextColumn(df):
    textCols = ['Name', 'Location']
    for textCol in textCols:
        if textCol in df.columns:
            df = df.drop(textCol, axis=1)
    return df

def getLocation():
    merge_result = pd.read_excel("../shanghanlun_mergeOCR_afterExpansion.xlsx", header=0, index_col=0)
    volumeName = {4971387: '仲景全書卷一', 4971388: "仲景全書卷二", 4971389: "仲景全書卷三"}
    df = dict()
    df['Name'] = merge_result.index.copy()
    df['Location'] = [f"{volumeName[volumeId]} 第{startPage}頁"
                      for (volumeId, startPage) in merge_result[['VolumeID', 'StartPage']].values.tolist()]
    df = pd.DataFrame.from_dict(df)
    return df

herbs_frequency_barChart = px.bar(
    herbs_frequency_df,
    x='藥材名稱',
    y='出現次數',
    title='Frequencies of herbs in Shang-Han-Lun',
    orientation='v',
    labels={'藥材名稱': 'herb name', '出現次數': 'Frequency (times)'}
)

def getFormulaeDistribution():
    volumeName = {4971387: '仲景全書卷一', 4971388: "仲景全書卷二", 4971389: "仲景全書卷三"}

    def volumeId_to_name(id):
        return volumeName[id]

    df = pd.read_excel("../shanghanlun_mergeOCR_afterExpansion.xlsx", header=0)
    df['VolumeID'] = df['VolumeID'].apply(volumeId_to_name)
    fig = px.sunburst(
        df,
        path=['VolumeID', 'Subchapter', 'Name'],
        hover_data={'StartPage': True},
        title='Formulae Distribution in Shang-Han-Lun (傷寒論)'
    )
    fig.update_layout(height=800, width=800)
    return fig

formulae_herbCount_chart = px.line(
    merge_result,
    x='Name',
    y='herbCount',
    title='Number of Herbs used by Formulae',
    markers=True
)
formulae_herbCount_chart.update_xaxes(title_text="")
maxCount = merge_result.loc[:, 'herbCount'].max(axis=None)
herbCount_rects = [('桂枝湯方', '四逆湯方', '#00CC96'), ('葛根湯方', '炙甘草湯方', '#636EF9'),
         ('大承氣湯方', '竹葉石膏湯方', '#EE553B')]
for start, end, color in herbCount_rects:
    formulae_herbCount_chart.add_shape(
        type='rect',
        x0=start,
        y0=0,
        x1=end,
        y1=maxCount,
        fillcolor=color,
        opacity=0.25,
    )

def get_parallel_plot_on_common_herbs(numeric_table, herbNum, overviewTitle, subplotTitle):
    numeric_table = numeric_table.loc[:, (numeric_table != 0).any(axis=0)]
    cols = list(numeric_table.columns)
    for col in ['Name', 'Location']:
        if col in cols:
            cols.remove(col)
    cols = cols[:herbNum]
    df_toScan = numeric_table.loc[:, cols]
    maxDosage = df_toScan.max(axis=None)

    plots = plotly.subplots.make_subplots(
        rows=len(cols),
        cols=1,
        shared_xaxes=True,
        y_title="Weight (g)",
    )
    overview = go.Figure(layout={'title': overviewTitle})

    rects = [('桂枝湯方', '四逆湯方', '#00CC96'), ('葛根湯方', '炙甘草湯方', '#636EF9'), ('大承氣湯方', '竹葉石膏湯方', '#EE553B')]
    for index, col in enumerate(cols):
        fig = go.Scatter(
            x=numeric_table.loc[:, 'Name'],
            y=numeric_table.loc[:, col],
            mode='lines+markers+text',
            name=f'{col}'
        )
        plots.add_trace(
            fig,
            row=1 + index,
            col=1,
        )
        upper_limit = max(numeric_table.loc[:, col])
        for start, end, color in rects:
            plots.add_shape(
                type='rect',
                x0=start,
                y0=0,
                x1=end,
                y1=upper_limit,
                fillcolor=color,
                opacity=0.25,
                row=1 + index,
                col=1,
            )

        overview.add_trace(fig)

    plots.update_layout(
        height=1200,
        width=600,
        title_text=subplotTitle
    )
    for start, end, color in rects:
        overview.add_shape(
            type='rect',
            x0=start,
            y0=0,
            x1=end,
            y1=maxDosage,
            fillcolor=color,
            opacity=0.25,
        )

    return overview, plots


default_overview, default_stack_trend = get_parallel_plot_on_common_herbs(
    quantity_model,
    10,
    overviewTitle="Overview of dosage trends for the 10 most common herbs",
    subplotTitle="Stacked subplots with dosage trends"
)

def getPairPlot(dim=10):
    if dim <= 0:
        return None

    df = pd.read_excel("../clean_quantity_model.xlsx", header=0, index_col=0)
    herb_names = df.columns[:dim]
    labels = [herb_name for herb_name in herb_names]
    fig = px.scatter_matrix(
        df,
        dimensions=herb_names,
        hover_name=df.index,
        title=f'Compare and Contrast the {dim} most common herbs by dosage',
        height=800,
        labels=dict(zip(herb_names, labels))
    )
    fig.update_traces(diagonal_visible=False, showupperhalf=False)

    return fig


def get_heatmap(df,
                labels,
                precomputed=False,
                title="Differences across formulae",
                description="The Darker, The More Similar",
                x='Formula 1',
                y='Formula 2'
                ):
    if not precomputed:
        df = scale_df(df, None, MinMaxScaler())
        dist = pairwise.euclidean_distances(df)
        df = pd.DataFrame(dist, index=labels, columns=labels)

    title = title
    description = f"<br><sup>{description}</sup>"
    fig = px.imshow(
        df,
        labels=dict(x=x, y=y, color="Relative Distances"),
        x=labels,
        y=labels,
        title=title + description
    )
    if len(df.index) > 24:
        fig.update_layout(height=800)
    else:
        fig.update_layout(autosize=True)
    return fig

pair_plot = getPairPlot(dim=10)
HerbCorrHeatmap = get_heatmap(
    1 - round(quantity_model.iloc[:, 1:].corr(), 2),
    quantity_model.columns[1:],
    precomputed=True,
    title="Differences across herbs",
    x='Herb 1',
    y='Herb 2'
)

def getHerbLevel():
    df = pd.read_excel("../shanghanlun_mergeOCR_afterExpansion.xlsx", header=0)
    one_hot_model = pd.read_excel("../one_hot_model.xlsx", header=0, index_col=0)

    # Aim: find the grandparent and parent of each formula
    df['parent'] = df['dependOn']
    df['grandParent'] = '無'
    for index, data in df.iterrows():
        if data['dependOn'] != '無':
            parent = data['parent']
            grandParent = df[df['Name'] == parent]
            df.at[index, 'grandParent'] = grandParent.loc[grandParent.index[0], 'parent']

    # Aim: find the herb difference between a formula and its origin
    df['added'] = '不適用'
    df['removed'] = '不適用'
    for index, data in df.iterrows():
        if data['parent'] == '無':
            continue

        diff_herbs = one_hot_model.loc[data['Name']] - one_hot_model.loc[data['parent']]
        added_herbs = diff_herbs[diff_herbs == 1].index.values
        removed_herbs = diff_herbs[diff_herbs == -1].index.values

        df.at[index, 'added'] = ",".join(added_herbs) if len(added_herbs) > 0 else "無"
        df.at[index, 'removed'] = ",".join(removed_herbs) if len(removed_herbs) > 0 else "無"

    volumeName = {4971387: '仲景全書卷一', 4971388: "仲景全書卷二", 4971389: "仲景全書卷三"}
    names = ["Formulae hierarchy"] + df['Name'].tolist()
    parents = [""] + ["Formulae hierarchy" if source == '無' else source for source in df['parent']]
    formula_locations = ["不適用"] + \
                        [
                            f"{volumeName[volumeId]} 第{startPage}頁"
                            for (volumeId, startPage) in df[['VolumeID', 'StartPage']].values.tolist()
                        ]
    added_herbs = ["不適用"] + df['added'].tolist()
    removed_herbs = ["不適用"] + df['removed'].tolist()
    custom_data = [
        f"Location: {location}<br>Parent: {parent}<br>added: {added}<br>removed: {removed}"
        for (location, parent, added, removed) in zip(formula_locations, parents, added_herbs, removed_herbs)
    ]

    fig = go.Figure(go.Treemap(
        labels=names,
        parents=parents,
        hovertemplate=
        "<b>%{label}</b><br>" +
        '%{customdata}' +
        "<extra></extra>"
        ,
        customdata=custom_data,
    ))
    fig.update_layout(margin=dict(t=50, l=20, r=20, b=20))

    return fig

treemap = getHerbLevel()

Here is the code for clustering. Since it takes a long time to perform 10 times clustering. We choose to run it once and save it in json file. This could shorten the first response time of the website.

def kmeansVisualization():
    model_to_cluster = pd.read_excel("../clean_quantity_model.xlsx", header=0, index_col=0)

    fit = umap.UMAP()
    u = fit.fit_transform(model_to_cluster)
    u = pd.DataFrame(u, index=model_to_cluster.index, columns=['component 1', 'component 2'])

    df = pd.read_excel("../shanghanlun_mergeOCR_afterExpansion.xlsx", header=0, index_col=0)
    selected_merge_result = df.loc[model_to_cluster.index.values, :]

    volumeName = {4971387: '仲景全書卷一', 4971388: "仲景全書卷二", 4971389: "仲景全書卷三"}
    formula_location = [f"{volumeName[volumeId]} 第{startPage}頁" for (volumeId, startPage) in
                        selected_merge_result[['VolumeID', 'StartPage']].values.tolist()]

    figs = []
    cluster_tables = []
    sse = {}
    silhouette_scores = []

    # Perform k-means clustering
    for k in range(2, 11):
        kmeans = KMeans(init='random', n_clusters=k, random_state=42, n_init='auto')
        cluster_assignments = kmeans.fit_predict(model_to_cluster)
        labels = [f"Cluster_{label + 1}" for label in cluster_assignments]

        # record metrics
        sse[k] = kmeans.inertia_  # Inertia: Sum of distances of samples to their closest cluster center
        silhouette = silhouette_score(model_to_cluster, cluster_assignments, metric='euclidean')
        silhouette_scores.append(silhouette)

        # Plot the clusters
        fig = px.scatter(u, x='component 1', y='component 2', color=labels, hover_name=u.index,
                         hover_data={'component 1': False, 'component 2': False, 'location': formula_location},
                         title=f'K-Means Clustering (K={k}) - UMAP')
        figs.append(fig)

        clusters = {
            'Clusters': [f"Cluster {cluster_label + 1}" for cluster_label in range(k)],
            'Formulae': []
        }
        for cluster_label in range(k):
            cluster_indices = np.where(cluster_assignments == cluster_label)[0]
            cluster_data_indices = sorted(model_to_cluster.index[cluster_indices], key=lambda name: len(name))
            clusters['Formulae'].append(" ".join(cluster_data_indices))
        cluster_tables.append(clusters)

    def to_records(t):
        new_df = pd.DataFrame.from_dict(t)
        return new_df.to_dict('records')

    cluster_tables = list(map(to_records, cluster_tables))

    sse_chart = px.line(
        x=sse.keys(),
        y=sse.values(),
        title="Sum of squared distance for different k",
        markers=True,
        labels={
            "x": "k",
            "y": "Sum of squared distance"
        }
    )
    silhouette_chart = px.line(
        x=range(2,11),
        y=silhouette_scores,
        title="Silhouette Scores for different k",
        markers=True,
        labels={
            "x": "k",
            "y": "Silhouette Score"
        }
    )
    return figs, cluster_tables, sse_chart, silhouette_chart

def figure_to_json(figure, fileName):
    with open(fileName, "w") as f:
        plotly.io.write_json(figure, f)


if __name__ == '__main__':
    kmeans_scatter_plots, kmeans_cluster_tables, sse_lineChart, silhouette_lineChart = kmeansVisualization()

    for index, cluster in zip(range(2, 11), kmeans_scatter_plots):
        figure_to_json(cluster, f"./figures/cluster_{index}.json")
    for index, table in zip(range(2, 11), kmeans_cluster_tables):
        with open(f"./figures/cluster_table_{index}.json", "w") as out:
            json.dump(table, out)

    figure_to_json(sse_lineChart, "./figures/sse_lineChart.json")
    figure_to_json(silhouette_lineChart, "./figures/silhouette_lineChart.json")

################## Get json on website ##################
def getFormulaeClusterCharts():
    cluster_charts = []
    for k in range(2, 11):
        cluster_charts.append(plotly.io.read_json(f"./figures/cluster_{k}.json"))
    return cluster_charts


def getFormulaeClusterTables():
    cluster_tables = []
    for k in range(2, 11):
        with open(f"./figures/cluster_table_{k}.json", "r") as table_json:
            cluster_tables.append(json.load(table_json))
    return cluster_tables


kmeans_scatter_plot, kmeans_cluster_table = getFormulaeClusterCharts(), getFormulaeClusterTables()
sse_lineChart, silhouette_lineChart = \
    plotly.io.read_json("./figures/sse_lineChart.json"), \
        plotly.io.read_json("./figures/silhouette_lineChart.json")

Here is the code for the heatmap

formula_heatmap = get_heatmap(
    removeTextColumn(quantity_model),
    labels=quantity_model.loc[:, 'Name'],
)