{ "cells": [ { "cell_type": "markdown", "id": "116bcc5c", "metadata": {}, "source": [ "# Benchmarking\n", "\n", "In this section, we benchmark `GaugeFixer`'s computational performance for gauge-fixing computation in comparison with direct multiplication with the dense projection matrix.\n" ] }, { "cell_type": "code", "execution_count": 1, "id": "23db982a", "metadata": {}, "outputs": [], "source": [ "import tracemalloc\n", "from time import time\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "from itertools import product\n", "from matplotlib.ticker import LogLocator, NullFormatter\n", "from gaugefixer import AllOrderModel, PairwiseModel" ] }, { "cell_type": "markdown", "id": "ab3054d0", "metadata": {}, "source": [ "### Model configurations\n", "\n", "For simplicity, we will evaluate the performance on all-order and pairwise interaction models defined over protein sequences of different lengths. " ] }, { "cell_type": "code", "execution_count": 2, "id": "1ef8bc62", "metadata": {}, "outputs": [], "source": [ "results = []\n", "n_points = 12\n", "alphabet_name = \"protein\"\n", "models = [AllOrderModel, PairwiseModel]\n", "seq_lengths = [np.arange(2, 6), np.geomspace(3, 150, n_points).astype(int)]" ] }, { "cell_type": "markdown", "id": "f036b5a7", "metadata": {}, "source": [ "### Evaluate running time and memory usage" ] }, { "cell_type": "code", "execution_count": 3, "id": "7badb1fc", "metadata": {}, "outputs": [ { "data": { "application/vnd.microsoft.datawrangler.viewer.v0+json": { "columns": [ { "name": "index", "rawType": "int64", "type": "integer" }, { "name": "model", "rawType": "object", "type": "string" }, { "name": "n_features", "rawType": "int64", "type": "integer" }, { "name": "alphabet_name", "rawType": "object", "type": "string" }, { "name": "time", "rawType": "float64", "type": "float" }, { "name": "current_memory", "rawType": "float64", "type": "float" }, { "name": "peak_memory", "rawType": "float64", "type": "float" }, { "name": "dense_matrix", "rawType": "bool", "type": "boolean" } ], "ref": "3e036c7e-d1e2-43ad-8d6c-b17a009e960f", "rows": [ [ "0", "AllOrderModel", "441", "protein", "0.0016045570373535156", "0.010924", "0.048751", "False" ], [ "1", "AllOrderModel", "441", "protein", "0.0011653900146484375", "0.009152", "0.047243", "False" ], [ "2", "AllOrderModel", "441", "protein", "0.0010368824005126953", "0.009824", "0.047979", "False" ], [ "3", "AllOrderModel", "441", "protein", "0.0010228157043457031", "0.00928", "0.047371", "False" ], [ "4", "AllOrderModel", "441", "protein", "0.0016942024230957031", "0.009121", "0.047276", "False" ], [ "5", "AllOrderModel", "441", "protein", "0.0009927749633789062", "0.009088", "0.047243", "False" ], [ "6", "AllOrderModel", "441", "protein", "0.0009667873382568359", "0.009088", "0.047243", "False" ], [ "7", "AllOrderModel", "441", "protein", "0.0008757114410400391", "0.009088", "0.047243", "False" ], [ "8", "AllOrderModel", "441", "protein", "0.0009107589721679688", "0.009088", "0.047243", "False" ], [ "9", "AllOrderModel", "441", "protein", "0.001306772232055664", "0.009459", "0.047614", "False" ], [ "10", "AllOrderModel", "441", "protein", "0.003393411636352539", "0.012876", "3.209792", "True" ], [ "11", "AllOrderModel", "441", "protein", "0.00487828254699707", "0.0129", "3.209816", "True" ], [ "12", "AllOrderModel", "441", "protein", "0.003950595855712891", "0.0129", "3.209816", "True" ], [ "13", "AllOrderModel", "441", "protein", "0.005503177642822266", "0.0129", "3.209816", "True" ], [ "14", "AllOrderModel", "441", "protein", "0.00510716438293457", "0.0129", "3.209816", "True" ], [ "15", "AllOrderModel", "441", "protein", "0.007420539855957031", "0.012964", "3.209816", "True" ], [ "16", "AllOrderModel", "441", "protein", "0.005437374114990234", "0.012964", "3.209816", "True" ], [ "17", "AllOrderModel", "441", "protein", "0.005404949188232422", "0.012964", "3.209816", "True" ], [ "18", "AllOrderModel", "441", "protein", "0.00561070442199707", "0.012964", "3.209816", "True" ], [ "19", "AllOrderModel", "441", "protein", "0.005711555480957031", "0.012964", "3.209816", "True" ], [ "20", "AllOrderModel", "9261", "protein", "0.0030248165130615234", "0.15028", "0.774419", "False" ], [ "21", "AllOrderModel", "9261", "protein", "0.0030622482299804688", "0.150304", "0.774443", "False" ], [ "22", "AllOrderModel", "9261", "protein", "0.003171205520629883", "0.150368", "0.774443", "False" ], [ "23", "AllOrderModel", "9261", "protein", "0.002008199691772461", "0.150368", "0.774443", "False" ], [ "24", "AllOrderModel", "9261", "protein", "0.001842498779296875", "0.150368", "0.774443", "False" ], [ "25", "AllOrderModel", "9261", "protein", "0.0019376277923583984", "0.150368", "0.774443", "False" ], [ "26", "AllOrderModel", "9261", "protein", "0.0020265579223632812", "0.150368", "0.774443", "False" ], [ "27", "AllOrderModel", "9261", "protein", "0.0026235580444335938", "0.150368", "0.774443", "False" ], [ "28", "AllOrderModel", "9261", "protein", "0.002708911895751953", "0.150368", "0.774443", "False" ], [ "29", "AllOrderModel", "9261", "protein", "0.0018563270568847656", "0.150368", "0.774443", "False" ], [ "30", "AllOrderModel", "9261", "protein", "1.5646154880523682", "0.154044", "1374.057512", "True" ], [ "31", "AllOrderModel", "9261", "protein", "1.593005895614624", "0.154132", "1374.057536", "True" ], [ "32", "AllOrderModel", "9261", "protein", "1.5537986755371094", "0.155528", "1374.059192", "True" ], [ "33", "AllOrderModel", "9261", "protein", "1.4868297576904297", "0.154132", "1374.057536", "True" ], [ "34", "AllOrderModel", "9261", "protein", "1.392214059829712", "0.154132", "1374.057536", "True" ], [ "35", "AllOrderModel", "9261", "protein", "1.6476714611053467", "0.155404", "1374.057888", "True" ], [ "36", "AllOrderModel", "9261", "protein", "1.5982186794281006", "0.154428", "1374.057832", "True" ], [ "37", "AllOrderModel", "9261", "protein", "1.3910224437713623", "0.154132", "1374.057536", "True" ], [ "38", "AllOrderModel", "9261", "protein", "1.3454842567443848", "0.154577", "1374.057981", "True" ], [ "39", "AllOrderModel", "9261", "protein", "1.4789447784423828", "0.154744", "1374.058376", "True" ], [ "40", "AllOrderModel", "194481", "protein", "0.0207822322845459", "3.114688", "15.967235", "False" ], [ "41", "AllOrderModel", "194481", "protein", "0.021856069564819336", "3.114712", "15.967259", "False" ], [ "42", "AllOrderModel", "194481", "protein", "0.019771337509155273", "3.115118", "15.967601", "False" ], [ "43", "AllOrderModel", "194481", "protein", "0.028174400329589844", "3.114776", "15.967259", "False" ], [ "44", "AllOrderModel", "194481", "protein", "0.025847196578979492", "3.114776", "15.967259", "False" ], [ "45", "AllOrderModel", "194481", "protein", "0.023633241653442383", "3.114885", "15.967368", "False" ], [ "46", "AllOrderModel", "194481", "protein", "0.0224611759185791", "3.114776", "15.967259", "False" ], [ "47", "AllOrderModel", "194481", "protein", "0.023927927017211914", "3.114776", "15.967259", "False" ], [ "48", "AllOrderModel", "194481", "protein", "0.022050857543945312", "3.114272", "15.966755", "False" ], [ "49", "AllOrderModel", "194481", "protein", "0.02091503143310547", "3.113936", "15.966419", "False" ] ], "shape": { "columns": 7, "rows": 210 } }, "text/html": [ "
| \n", " | model | \n", "n_features | \n", "alphabet_name | \n", "time | \n", "current_memory | \n", "peak_memory | \n", "dense_matrix | \n", "
|---|---|---|---|---|---|---|---|
| 0 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001605 | \n", "0.010924 | \n", "0.048751 | \n", "False | \n", "
| 1 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001165 | \n", "0.009152 | \n", "0.047243 | \n", "False | \n", "
| 2 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001037 | \n", "0.009824 | \n", "0.047979 | \n", "False | \n", "
| 3 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001023 | \n", "0.009280 | \n", "0.047371 | \n", "False | \n", "
| 4 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001694 | \n", "0.009121 | \n", "0.047276 | \n", "False | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 205 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "5.179069 | \n", "71.571140 | \n", "295.822343 | \n", "False | \n", "
| 206 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "4.999057 | \n", "71.570496 | \n", "295.821699 | \n", "False | \n", "
| 207 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "5.332721 | \n", "71.571140 | \n", "295.822343 | \n", "False | \n", "
| 208 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "4.825775 | \n", "71.570496 | \n", "295.821699 | \n", "False | \n", "
| 209 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "5.127097 | \n", "71.571140 | \n", "295.822343 | \n", "False | \n", "
210 rows × 7 columns
\n", "| \n", " | model | \n", "n_features | \n", "alphabet_name | \n", "time | \n", "current_memory | \n", "peak_memory | \n", "dense_matrix | \n", "label | \n", "
|---|---|---|---|---|---|---|---|---|
| 0 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001605 | \n", "0.010924 | \n", "0.048751 | \n", "False | \n", "all-order (GaugeFixer) | \n", "
| 1 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001165 | \n", "0.009152 | \n", "0.047243 | \n", "False | \n", "all-order (GaugeFixer) | \n", "
| 2 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001037 | \n", "0.009824 | \n", "0.047979 | \n", "False | \n", "all-order (GaugeFixer) | \n", "
| 3 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001023 | \n", "0.009280 | \n", "0.047371 | \n", "False | \n", "all-order (GaugeFixer) | \n", "
| 4 | \n", "AllOrderModel | \n", "441 | \n", "protein | \n", "0.001694 | \n", "0.009121 | \n", "0.047276 | \n", "False | \n", "all-order (GaugeFixer) | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 205 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "5.179069 | \n", "71.571140 | \n", "295.822343 | \n", "False | \n", "pairwise (GaugeFixer) | \n", "
| 206 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "4.999057 | \n", "71.570496 | \n", "295.821699 | \n", "False | \n", "pairwise (GaugeFixer) | \n", "
| 207 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "5.332721 | \n", "71.571140 | \n", "295.822343 | \n", "False | \n", "pairwise (GaugeFixer) | \n", "
| 208 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "4.825775 | \n", "71.570496 | \n", "295.821699 | \n", "False | \n", "pairwise (GaugeFixer) | \n", "
| 209 | \n", "PairwiseModel | \n", "4473001 | \n", "protein | \n", "5.127097 | \n", "71.571140 | \n", "295.822343 | \n", "False | \n", "pairwise (GaugeFixer) | \n", "
210 rows × 8 columns
\n", "