Computation speed benchmark

In this notebook, we tested the average computation speed for various models in different cases.

import multiprocessing
import os
import platform
import re
import subprocess
import time

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from IPython.display import HTML, display

Local machine details

Log the local machine details on which the test is carried out:

def get_cpu_model(spec="model name"):
    if platform.system() == "Windows":
        return platform.processor()
    if platform.system() == "Darwin":
        os.environ["PATH"] = os.environ["PATH"] + os.pathsep + "/usr/sbin"
        command = "sysctl -n machdep.cpu.brand_string"
        return subprocess.check_output(command).strip()
    if platform.system() == "Linux":
        command = "cat /proc/cpuinfo"
        stream = os.popen(command)
        all_info = stream.read()
        all_info.strip()
        all_info = all_info.split("\n")
        # all_info = str(subprocess.check_output(command, shell=True).strip())
        for line in all_info:
            if spec in line:
                return re.sub(f".*{spec}.*: ", "", line, 1)

def get_specs():
    table = [
        # ["", platform.version()],
        ["Machine", platform.machine()],
        ["Platform", platform.platform()],
        ["Architecture", platform.architecture()],
        ["Cores", get_cpu_model("model name")],
        ["Number of cores", multiprocessing.cpu_count()],
        ["Python version", platform.python_version()],
    ]
    dataframe = pd.DataFrame(table, columns=["Spec", "Value"])
    display(HTML(dataframe.to_html(index=False)))

get_specs()

Spec	Value
Machine	x86_64
Platform	Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.29
Architecture	(64bit, ELF)
Cores	Intel(R) Core(TM) i5-10210U CPU @ 1.60GHz
Number of cores	8
Python version	3.8.10

Test function

Create a test function to test the models:

def test_model(Model, n):

    init_time = list()
    encode_time = list()
    decode_time = list()
    total_time = list()

    # Iterations for computing mean and standard deviation
    iterations = 50

    for i in range(1, iterations + 1):
        # Initialization
        start_process = time.time()
        model = Model(n, 1)
        # Encoding
        start_encode = time.time()
        for dim in range(model.n):
            model.encode(0.5, dim)
        # Decoding
        start_decode = time.time()
        result = model.decode()
        end_process = time.time()

        # Store timings
        init_time.append(start_encode - start_process)
        encode_time.append(start_decode - start_encode)
        decode_time.append(end_process - start_decode)
        total_time.append(end_process - start_process)

    return [
        n,
        np.mean(init_time),
        np.mean(encode_time),
        np.mean(decode_time),
        np.mean(total_time),
        np.std(total_time),
    ]

def plot_results(df, title="Model"):
    """Plot all the results"""

    # Total timewith STD
    fig = plt.figure(figsize=(15, 6), dpi=150)

    df.plot(
        x="n",
        y=[4],
        yerr="Total STD",
        uplims=True,
        lolims=True,
        kind="line",
        color="r",
        ax=plt.gca(),
    )
    plt.title(f"{title} - Total time with STD")

    plt.grid(visible=True, which="major", linestyle="-")
    plt.grid(visible=True, which="minor", linestyle="--", alpha=0.2)
    plt.minorticks_on()

    plt.show()

    # Timings insight
    fig = plt.figure(figsize=(15, 6), dpi=150)

    df.plot(x="n", y=[1, 2, 3, 4], kind="line", ax=plt.gca())
    plt.title(f"{title} - Timings insight")

    plt.grid(visible=True, which="major", linestyle="-")
    plt.grid(visible=True, which="minor", linestyle="--", alpha=0.2)
    plt.minorticks_on()

    plt.show()

    # Zooming in
    print("Zooming in:")
    fig = plt.figure(figsize=(15, 6), dpi=150)

    df.plot(x="n", y=[1, 2], kind="line", ax=plt.subplot(1, 2, 1))
    plt.title("Initialization and encoding only")

    plt.grid(visible=True, which="major", linestyle="-")
    plt.grid(visible=True, which="minor", linestyle="--", alpha=0.2)
    plt.minorticks_on()

    ax = plt.subplot(1, 2, 2)
    df[1:19].plot(x="n", y=[1, 2, 3], kind="line", ax=ax)
    df[1:19].plot(x="n", y=[4], yerr="Total STD", kind="line", ax=ax)
    plt.title("Showing results for n < 20")

    plt.grid(visible=True, which="major", linestyle="-")
    plt.grid(visible=True, which="minor", linestyle="--", alpha=0.2)
    plt.minorticks_on()

    plt.show()

AngularModel

We test the initialization, encode and decode time for a input of .5 on each dimension, for increasing dimensions (\(n \in [0,26]\))

from qrobot.models import AngularModel

max_n = 26

table = list()

for n in range(1, max_n + 1):
    print(f"Testing n={n}", end="\r")
    table.append(test_model(AngularModel, n))
print("             ")

df_angular = pd.DataFrame(
    table, columns=["n", "Initialization", "Encode", "Decode", "Total", "Total STD"]
)

Plotting the results:

plot_results(df_angular, "AngularModel")

Zooming in:

Numerical values:

df_angular

	n	Initialization	Encode	Decode	Total	Total STD
0	1	0.000090	0.000053	0.015489	0.015633	0.034521
1	2	0.000110	0.000083	0.016208	0.016402	0.004183
2	3	0.000098	0.000093	0.018067	0.018258	0.002224
3	4	0.000098	0.000104	0.021688	0.021890	0.002773
4	5	0.000103	0.000116	0.025970	0.026188	0.004132
5	6	0.000123	0.000142	0.033105	0.033370	0.011682
6	7	0.000128	0.000158	0.036914	0.037200	0.012023
7	8	0.000136	0.000175	0.041892	0.042203	0.008696
8	9	0.000140	0.000188	0.042933	0.043261	0.005541
9	10	0.000140	0.000202	0.047731	0.048074	0.009502
10	11	0.000141	0.000214	0.049028	0.049383	0.005541
11	12	0.000156	0.000216	0.052194	0.052566	0.011456
12	13	0.000139	0.000218	0.053215	0.053573	0.007068
13	14	0.000143	0.000251	0.058227	0.058620	0.006625
14	15	0.000283	0.000424	0.076272	0.076979	0.014698
15	16	0.000281	0.000459	0.079937	0.080677	0.014732
16	17	0.000312	0.000597	0.089898	0.090807	0.021098
17	18	0.000402	0.000589	0.103118	0.104109	0.020345
18	19	0.000341	0.000630	0.109812	0.110783	0.012404
19	20	0.000343	0.000631	0.134797	0.135771	0.015628
20	21	0.000347	0.000624	0.148700	0.149671	0.026392
21	22	0.000389	0.000656	0.198444	0.199489	0.018418
22	23	0.000393	0.000787	0.289993	0.291173	0.030890
23	24	0.000262	0.000449	0.506927	0.507638	0.038720
24	25	0.000244	0.000443	0.817398	0.818086	0.132657
25	26	0.000243	0.000459	1.510985	1.511687	0.226645

---

LinearModel

We test the initialization, encode and decode time for a input of .5 on each dimension, for increasing dimensions (\(n \in [0,26]\))

from qrobot.models import LinearModel

max_n = 26

table = list()

for n in range(1, max_n + 1):
    print(f"Testing n={n}", end="\r")
    table.append(test_model(LinearModel, n))
print("             ")


df_linear = pd.DataFrame(
    table, columns=["n", "Initialization", "Encode", "Decode", "Total", "Total STD"]
)

plot_results(df_linear, "LinearModel")

Zooming in:

df_linear

	n	Initialization	Encode	Decode	Total	Total STD
0	1	0.000092	0.000074	0.010906	0.011072	0.003508
1	2	0.000095	0.000099	0.015090	0.015284	0.012434
2	3	0.000096	0.000112	0.016888	0.017096	0.001992
3	4	0.000101	0.000129	0.021175	0.021405	0.002223
4	5	0.000121	0.000166	0.026563	0.026849	0.004640
5	6	0.000268	0.000403	0.081685	0.082355	0.031608
6	7	0.000165	0.000256	0.045953	0.046374	0.007647
7	8	0.000170	0.000277	0.048785	0.049232	0.006297
8	9	0.000161	0.000269	0.048968	0.049398	0.005945
9	10	0.000190	0.000321	0.062090	0.062601	0.009553
10	11	0.000132	0.000249	0.050055	0.050436	0.012080
11	12	0.000142	0.000270	0.054940	0.055352	0.007372
12	13	0.000156	0.000304	0.060936	0.061396	0.006280
13	14	0.000160	0.000316	0.062640	0.063117	0.011835
14	15	0.000276	0.000538	0.074725	0.075538	0.013014
15	16	0.000307	0.000583	0.078972	0.079863	0.014034
16	17	0.000337	0.000662	0.089452	0.090451	0.024715
17	18	0.000375	0.000713	0.089646	0.090735	0.017322
18	19	0.000393	0.000744	0.109232	0.110370	0.028380
19	20	0.000380	0.000745	0.104235	0.105361	0.023976
20	21	0.000383	0.000744	0.124019	0.125146	0.017510
21	22	0.000407	0.000744	0.159346	0.160497	0.024465
22	23	0.000411	0.000811	0.240353	0.241575	0.032933
23	24	0.000253	0.000547	0.416738	0.417538	0.028743
24	25	0.000217	0.000478	0.683482	0.684177	0.028145
25	26	0.000247	0.000542	1.410785	1.411574	0.180279

Comparison

fig = plt.figure(figsize=(15, 6), dpi=150)

df_angular.plot(x="n", y=[4], kind="line", ax=plt.gca())
df_linear.plot(x="n", y=[4], kind="line", ax=plt.gca())
plt.legend(["AngularModel", "LinearModel"])
plt.title("Total times comparison")

plt.grid(visible=True, which="major", linestyle="-")
plt.grid(visible=True, which="minor", linestyle="--", alpha=0.2)
plt.minorticks_on()

plt.show()

fig = plt.figure(figsize=(15, 6), dpi=150)

df_angular.plot(x="n", y=[5], kind="line", ax=plt.gca())
df_linear.plot(x="n", y=[5], kind="line", ax=plt.gca())

plt.legend(["AngularModel", "LinearModel"])
plt.title("Total times STD comparison")

plt.grid(visible=True, which="major", linestyle="-")
plt.grid(visible=True, which="minor", linestyle="--", alpha=0.2)
plt.minorticks_on()

plt.show()

fig = plt.figure(figsize=(15, 6), dpi=150)

df_angular[1:19].plot(x="n", y=[4], kind="line", ax=plt.gca())
df_linear[1:19].plot(x="n", y=[4], kind="line", ax=plt.gca())

plt.legend(["AngularModel", "LinearModel"])
plt.title("Total times comparison (n < 20)")

plt.grid(visible=True, which="major", linestyle="-")
plt.grid(visible=True, which="minor", linestyle="--", alpha=0.2)
plt.minorticks_on()

plt.show()