René's URL Explorer Experiment

{"@context":"https://schema.org","@type":"DiscussionForumPosting","headline":"[Bug]: Memory leak from repeated live plotting","articleBody":"### Bug summary\r\n\r\nMemory leak from repeated live plotting: seems to be caused by `transform._parents` growing continuously over time, as in #11956 and #11972.\r\n\r\n### Code for reproduction\r\n\r\n```python\r\nimport datetime\r\n#import tkinter for GUI\r\nimport tkinter as tk\r\nfrom tkinter import W, LEFT\r\n#font types\r\nTITLE_FONT = (\"Verdana\", 14, 'bold')\r\nLARGE_FONT = (\"Verdana\", 12)\r\nMEDIUM_FONT = (\"Verdana\", 10)\r\nSMALL_FONT = (\"Verdana\", 8)\r\n#import stuff for graph\r\nimport matplotlib\r\nfrom matplotlib import ticker as mticker\r\nfrom matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk \r\nmatplotlib.use('TkAgg')\r\nfrom matplotlib import figure\r\nfrom matplotlib import dates as mdates\r\n#import animation to make graph live\r\nimport matplotlib.animation as animation\r\nfrom matplotlib import style\r\nstyle.use(\"seaborn-darkgrid\")\r\nimport random as r\r\nnum_contacts = 5\r\n\r\nimport cProfile\r\nimport pstats\r\nimport io\r\n\r\n#create figure for plots and set figure size/layout\r\nf = figure.Figure(figsize=(16.6,15), dpi=100, facecolor='white')\r\nf.subplots_adjust(top=0.993, bottom=0.015, left=0.04, right = 0.96, hspace=0.65)\r\n\r\nparam_dict = {}\r\nparam_list = ['pH', 'TDS (ppm)', 'Rela. Humidity (%)', 'Air Temp (\\N{DEGREE SIGN}C)', 'Water Temp (\\N{DEGREE SIGN}C)', 'Water Level (cm)']\r\nparam_ylim = [(0, 15), (0, 15), (0, 15), (0, 15), (0, 15), (0, 15)]\r\nlive_dict = {}\r\n\r\n\r\nclass Live_Text:\r\n    def __init__(self, label):\r\n        self.label = label\r\n    \r\nclass Sensor_Plot:\r\n    def __init__(self, plot, tList, x_ax, ylim, param, incoming_data, plot_color):\r\n        self.plot = plot\r\n        self.tList = tList\r\n        self.x_ax = x_ax\r\n        self.ylim = ylim\r\n        self.param = param\r\n        self.incoming_data = incoming_data #\u003c- graph is bound by incoming data and Data Summary Table displays most recent value 20 of them\r\n        self.plot_color = plot_color #initially 'b' for all\r\n        \r\n    def make_plot(self):\r\n        self.plot.clear()\r\n        self.plot.set_xlabel('Time')\r\n        self.plot.set_ylabel(self.param)\r\n        self.plot.set_ylim(self.ylim)\r\n\r\n        self.x_ax.xaxis_date()\r\n        self.x_ax.xaxis.set_major_formatter(mdates.DateFormatter('%a %I:%M:%S %p'))\r\n        \r\n        [tk.set_visible(True) for tk in self.x_ax.get_xticklabels()]\r\n        [label.set_rotation(10) for label in self.x_ax.xaxis.get_ticklabels()] #slant the x axis tick labels for extra coolness\r\n\r\n        if len(self.tList) \u003e 4:\r\n            self.x_ax.set_xlim(self.tList[-2], self.tList[0])\r\n        self.x_ax.xaxis.set_major_locator(mticker.MaxNLocator(nbins = 4))\r\n        \r\n        self.plot.fill_between(self.tList, self.incoming_data, #where=(self.incoming_data \u003e [0]*len(self.incoming_data))\r\n                               facecolor=self.plot_color, edgecolor=self.plot_color, alpha=0.5) #blue @initilization\r\n\r\ndef initialize_plots(): #intiailizes plots...\r\n    global initialize_plots\r\n    try:\r\n        most_recent = [(int(round(datetime.datetime.now().timestamp())),) + tuple([r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10)])]\r\n        for i, param in enumerate(param_list, 1):\r\n            tList = []\r\n            most_recent_any_size = []\r\n            for j in range(len(most_recent)):\r\n                #time_f = datetime.strptime(most_recent[j][0], \"%m/%d/%Y %H:%M:%S\")\r\n                time_f = datetime.datetime.fromtimestamp(most_recent[j][0])\r\n                tList.append(time_f)\r\n                most_recent_any_size.append(most_recent[j][i])\r\n\r\n            subplot = f.add_subplot(6, 2, i)  # sharex?\r\n            x_ax = f.get_axes()\r\n            \r\n            current_plot = Sensor_Plot(subplot, tList, x_ax[i-1], param_ylim[i-1], param, most_recent_any_size, 'b')\r\n            param_dict[param] = current_plot\r\n            current_plot.make_plot()\r\n                    \r\n    except: #if there is no data points available to plot, initialize the subplots\r\n        for i, param in enumerate(param_list, 1):\r\n            subplot = f.add_subplot(6, 2, i)\r\n            x_ax = f.get_axes()\r\n            current_plot = Sensor_Plot(subplot, [], x_ax[i-1], param_ylim[i-1], param, [], 'b')\r\n            param_dict[param] = current_plot\r\n            #current_plot.make_plot()    \r\n    #reader.commit()\r\n    initialize_plots = _plots_initialized\r\n\r\ndef _plots_initialized(): #ensures plots only intialized once though!\r\n    pass\r\ninitialize_plots()\r\n\r\n\r\n\r\n###ANIMATE FUNCTION, REMOVE LAST ITEM FROM MOST_RECENT_ANY LIST AND INSERT FRESHLY CALLED VALUE TO BE FIRST IN LIST\r\ndef animate(ii):\r\n    profile = cProfile.Profile()\r\n    profile.enable()\r\n\r\n    while True:\r\n        most_recent_time_graphed = param_dict[param_list[0]] #first, pulls up first plot\r\n        #most_recent = reader.query_by_num(table=\"SensorData\", num=1)\r\n        #generate fake sensor data here\r\n        most_recent = [(int(round(datetime.datetime.now().timestamp())),) + tuple([r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10)])] #log time in unix as int\r\n        print(most_recent)\r\n        #reader.commit()         #if identical, do not animate\r\n        #then checks that plot's time list\r\n        if  (len(most_recent) == 0):\r\n            break\r\n        \r\n        #time_reader = datetime.strptime(most_recent[0][0], \"%m/%d/%Y %H:%M:%S\")\r\n        time_reader = datetime.datetime.fromtimestamp(most_recent[0][0])\r\n        if (len(most_recent_time_graphed.tList) != 0) and (time_reader == most_recent_time_graphed.tList[0]):\r\n            for i, param in enumerate(param_list, 1):\r\n                current_text = live_dict[param]\r\n                current_text.label.config(text=most_recent[0][i], fg=\"black\", bg=\"white\")\r\n            break #checks if the timestamp is exactly the same as prior, i.e. no new data points have been logged in this frame\r\n        #do I have to add an else?\r\n    \r\n        else:\r\n            for i, key in enumerate(param_dict, 1):\r\n                current_plot = param_dict[key]\r\n                current_param_val = float(most_recent[0][i])\r\n                current_text = live_dict[key] #update to live text data summary\r\n\r\n\r\n                current_text.label.config(text=most_recent[0][i], fg=\"black\", bg=\"white\")\r\n                current_plot.plot_color = 'g'\r\n\r\n                data_stream = current_plot.incoming_data\r\n                time_stream = current_plot.tList\r\n                data_stream.insert(0, most_recent[0][i])\r\n                #time_f = datetime.strptime(most_recent[0][0], \"%m/%d/%Y %H:%M:%S\")\r\n                time_f = datetime.datetime.fromtimestamp(most_recent[0][0])\r\n                time_stream.insert(0, time_f)\r\n                if len(data_stream) \u003c 20: #graph updates, growing to show 20 points\r\n                    current_plot.make_plot()\r\n                else:                      #there are 20 points and more available, so animation occurs\r\n                    data_stream.pop()\r\n                    time_stream.pop()\r\n                    current_plot.make_plot()\r\n            break\r\n    if time_reader.minute == 50 and most_recent_time_graphed.tList[1].minute == 49:\r\n        profile.disable()\r\n        result = io.StringIO()\r\n        pstats.Stats(profile,stream=result).print_stats()\r\n        result=result.getvalue()\r\n        # chop the string into a csv-like buffer\r\n        result='ncalls'+result.split('ncalls')[-1]\r\n        result='\\n'.join([','.join(line.rstrip().split(None,5)) for line in result.split('\\n')])\r\n        # save it to disk\r\n        with open('testing.csv', 'r') as f:\r\n            result = f.read() + 'hour: ' + str(time_reader.hour) + '\\n' + result\r\n        \r\n        with open('testing.csv', 'w+') as f:\r\n            f.write(result)\r\n            f.close()\r\n    else:\r\n        profile.disable()\r\n                           \r\n#initialization\r\nclass AllWindow(tk.Tk):\r\n    def __init__(self, *args, **kwargs):\r\n        tk.Tk.__init__(self, *args, **kwargs)\r\n        #add title\r\n        tk.Tk.wm_title(self, \"Matplotlib Live Plotting\")\r\n        container = tk.Frame(self)\r\n        container.pack(side=\"top\", fill=\"both\", expand=True)\r\n        container.grid_rowconfigure(0, weight=1)\r\n        container.grid_columnconfigure(0, weight=1)\r\n        #show the frames\r\n        self.frames = {}\r\n        #remember to add page to this list when making new ones\r\n        frame = HomePage(container, self)\r\n        #set background color for the pages\r\n        frame.config(bg='white')\r\n        self.frames[HomePage] = frame\r\n        frame.grid(row=0, column=0, sticky=\"nsew\")\r\n        self.show_frame(HomePage)\r\n    def show_frame(self, cont):\r\n        frame = self.frames[cont]\r\n        frame.tkraise()\r\n    #end program fcn triggered by quit button\r\n    def die(self):\r\n        exit()\r\n\r\n#add home page\r\nclass HomePage(tk.Frame):\r\n    def __init__(self, parent, controller):\r\n        tk.Frame.__init__(self,parent)\r\n        canvas = FigureCanvasTkAgg(f, self)\r\n        #background = canvas.copy_from_bbox(f.bbox)\r\n        canvas.draw()\r\n        #embed graph into canvas\r\n        canvas.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand = True)\r\n        #add navigation bar\r\n        toolbar = NavigationToolbar2Tk(canvas, self)\r\n        toolbar.update()\r\n        #data table labels\r\n        for i, param in enumerate(param_list): #tk.Label self refers to Homepage\r\n            param_label = tk.Label(self, text=param, fg=\"black\", bg=\"white\",\r\n                            font = MEDIUM_FONT, borderwidth = 2, relief = \"ridge\",\r\n                            width=16, height=1, anchor=W, justify=LEFT)\r\n            param_label.place(x=5, y=500+22*i)\r\n\r\n        for i, param in enumerate(param_list):\r\n            loading_text = tk.Label(self, text=\"Loading\", fg=\"black\", bg=\"white\",\r\n                    font = MEDIUM_FONT, borderwidth = 2, relief = \"ridge\",\r\n                    width=7, height=1)\r\n            loading_text.place(x=140, y=500+22*i)\r\n            current_text = Live_Text(loading_text)\r\n            live_dict[param] = current_text\r\n    \r\n\r\napp = AllWindow()\r\n#app.geometry('1280x623')\r\napp.geometry('1917x970')\r\n#update animation first\r\nani = animation.FuncAnimation(f, animate, interval=100)\r\n#mainloop\r\napp.mainloop()\r\n```\r\n\r\n### Actual outcome\r\n\r\nThe program eventually crashes after running for several days. Memory usage gradually increases throughout the program runtime. Additionally, profiling the animate function revealed that `_invalidate_internal` in Matplotlib seemed to be growing in its execution time. This is in line with #11956 and #11972, since `_invalidate_internal` is searching through `transform._parents`, which grows over time. However, these issues were resolved, so I am unsure why I am encountering the same problem.\r\n\r\n### Expected outcome\r\n\r\nExpect the program to run continuously and indefinitely without memory usage increasing over time.\r\n\r\n### Operating system\r\n\r\nRaspbian GNU/Linux 10 (buster)\r\n\r\n### Matplotlib Version\r\n\r\n3.4.3\r\n\r\n### Matplotlib Backend\r\n\r\nQt5Agg\r\n\r\n### Python version\r\n\r\n3.9\r\n\r\n### Jupyter version\r\n\r\n_No response_\r\n\r\n### Other libraries\r\n\r\n_No response_\r\n\r\n### Installation\r\n\r\npip\r\n\r\n### Conda channel\r\n\r\n_No response_","author":{"url":"https://github.com/bencaterine","@type":"Person","name":"bencaterine"},"datePublished":"2021-11-11T02:18:21.000Z","interactionStatistic":{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":7},"url":"https://github.com/21595/matplotlib/issues/21595"}