{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Using dill to pickle anything" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "ipyparallel doesn't do much in the way of serialization.\n", "It has custom zero-copy handling of numpy arrays,\n", "but other than that, it doesn't do anything other than the bare minimum to make basic interactively defined functions and classes sendable.\n", "\n", "There are a few projects that extend pickle to make just about anything sendable, and one of these is [dill](https://dill.readthedocs.io).\n", "Another is [cloudpickle](https://github.com/cloudpipe/cloudpickle).\n", "\n", "To install dill:\n", " \n", " pip install dill" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "First, as always, we create a task function, this time with a closure" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "def make_closure(a):\n", " \"\"\"make a weird function with a closure on an open file, and return it\"\"\"\n", " import os\n", "\n", " f = open('/tmp/dilltest', 'a')\n", "\n", " def has_closure(b):\n", " product = a * b\n", " f.write(f\"{os.getpid()}: {product:g}\\n\")\n", " f.flush()\n", " return product\n", "\n", " return has_closure" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "!rm -f /tmp/dilltest\n", "closed = make_closure(5)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "closed(2)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "25810: 10\n" ] } ], "source": [ "!cat /tmp/dilltest" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "import pickle" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Without help, pickle can't deal with closures" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "ename": "AttributeError", "evalue": "Can't pickle local object 'make_closure..has_closure'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mpickle\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdumps\u001b[49m\u001b[43m(\u001b[49m\u001b[43mclosed\u001b[49m\u001b[43m)\u001b[49m\n", "\u001b[0;31mAttributeError\u001b[0m: Can't pickle local object 'make_closure..has_closure'" ] } ], "source": [ "pickle.dumps(closed)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "But after we import dill, magic happens" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "import dill" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "b'\\x80\\x04\\x95E\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x8c\\ndill._dill\\x94\\x8c\\x10_create_function\\x94\\x93\\x94(h\\x00\\x8c\\x0c_create_code\\x94\\x93...'" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dill.dumps(closed)[:64] + b'...'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "So from now on, pretty much everything is pickleable." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Using dill in IPython Parallel" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As usual, we start by creating our Client and View" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Starting 2 engines with \n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "d2ece3d105ca4c488a76246625ef8962", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/2 [00:00 1\u001b[0m \u001b[43mview\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply_sync\u001b[49m\u001b[43m(\u001b[49m\u001b[43mclosed\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/dev/ip/parallel/ipyparallel/client/view.py:228\u001b[0m, in \u001b[0;36mView.apply_sync\u001b[0;34m(self, _View__ipp_f, *args, **kwargs)\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mapply_sync\u001b[39m(\u001b[38;5;28mself\u001b[39m, __ipp_f, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 225\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"calls ``f(*args, **kwargs)`` on remote engines in a blocking manner,\u001b[39;00m\n\u001b[1;32m 226\u001b[0m \u001b[38;5;124;03m returning the result.\u001b[39;00m\n\u001b[1;32m 227\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 228\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_really_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43m__ipp_f\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mblock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/.virtualenvs/ipp7-stable/lib/python3.11/site-packages/decorator.py:232\u001b[0m, in \u001b[0;36mdecorate..fun\u001b[0;34m(*args, **kw)\u001b[0m\n\u001b[1;32m 230\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kwsyntax:\n\u001b[1;32m 231\u001b[0m args, kw \u001b[38;5;241m=\u001b[39m fix(args, kw, sig)\n\u001b[0;32m--> 232\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcaller\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mextras\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/dev/ip/parallel/ipyparallel/client/view.py:55\u001b[0m, in \u001b[0;36msync_results\u001b[0;34m(f, self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_in_sync_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 54\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 55\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 56\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_in_sync_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n", "File \u001b[0;32m~/.virtualenvs/ipp7-stable/lib/python3.11/site-packages/decorator.py:232\u001b[0m, in \u001b[0;36mdecorate..fun\u001b[0;34m(*args, **kw)\u001b[0m\n\u001b[1;32m 230\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kwsyntax:\n\u001b[1;32m 231\u001b[0m args, kw \u001b[38;5;241m=\u001b[39m fix(args, kw, sig)\n\u001b[0;32m--> 232\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcaller\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mextras\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/dev/ip/parallel/ipyparallel/client/view.py:39\u001b[0m, in \u001b[0;36msave_ids\u001b[0;34m(f, self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 37\u001b[0m n_previous \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclient\u001b[38;5;241m.\u001b[39mhistory)\n\u001b[1;32m 38\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 39\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 41\u001b[0m nmsgs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclient\u001b[38;5;241m.\u001b[39mhistory) \u001b[38;5;241m-\u001b[39m n_previous\n", "File \u001b[0;32m~/dev/ip/parallel/ipyparallel/client/view.py:1364\u001b[0m, in \u001b[0;36mLoadBalancedView._really_apply\u001b[0;34m(self, f, args, kwargs, block, track, after, follow, timeout, targets, retries)\u001b[0m\n\u001b[1;32m 1359\u001b[0m follow \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_render_dependency(follow)\n\u001b[1;32m 1360\u001b[0m metadata \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(\n\u001b[1;32m 1361\u001b[0m after\u001b[38;5;241m=\u001b[39mafter, follow\u001b[38;5;241m=\u001b[39mfollow, timeout\u001b[38;5;241m=\u001b[39mtimeout, targets\u001b[38;5;241m=\u001b[39midents, retries\u001b[38;5;241m=\u001b[39mretries\n\u001b[1;32m 1362\u001b[0m )\n\u001b[0;32m-> 1364\u001b[0m future \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend_apply_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1365\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_socket\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrack\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrack\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmetadata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmetadata\u001b[49m\n\u001b[1;32m 1366\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1368\u001b[0m ar \u001b[38;5;241m=\u001b[39m AsyncResult(\n\u001b[1;32m 1369\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclient,\n\u001b[1;32m 1370\u001b[0m future,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1373\u001b[0m owner\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 1374\u001b[0m )\n\u001b[1;32m 1375\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m block:\n", "File \u001b[0;32m~/dev/ip/parallel/ipyparallel/client/client.py:1954\u001b[0m, in \u001b[0;36mClient.send_apply_request\u001b[0;34m(self, socket, f, args, kwargs, metadata, track, ident, message_future_hook)\u001b[0m\n\u001b[1;32m 1951\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(metadata, \u001b[38;5;28mdict\u001b[39m):\n\u001b[1;32m 1952\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata must be dict, not \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(metadata)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1954\u001b[0m bufs \u001b[38;5;241m=\u001b[39m \u001b[43mserialize\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpack_apply_message\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1955\u001b[0m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1956\u001b[0m \u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1957\u001b[0m \u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1958\u001b[0m \u001b[43m \u001b[49m\u001b[43mbuffer_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbuffer_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1959\u001b[0m \u001b[43m \u001b[49m\u001b[43mitem_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitem_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1960\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1962\u001b[0m future \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_send(\n\u001b[1;32m 1963\u001b[0m socket,\n\u001b[1;32m 1964\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapply_request\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1970\u001b[0m message_future_hook\u001b[38;5;241m=\u001b[39mmessage_future_hook,\n\u001b[1;32m 1971\u001b[0m )\n\u001b[1;32m 1972\u001b[0m msg_id \u001b[38;5;241m=\u001b[39m future\u001b[38;5;241m.\u001b[39mmsg_id\n", "File \u001b[0;32m~/dev/ip/parallel/ipyparallel/serialize/serialize.py:182\u001b[0m, in \u001b[0;36mpack_apply_message\u001b[0;34m(f, args, kwargs, buffer_threshold, item_threshold)\u001b[0m\n\u001b[1;32m 174\u001b[0m kwarg_bufs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\n\u001b[1;32m 175\u001b[0m chain\u001b[38;5;241m.\u001b[39mfrom_iterable(\n\u001b[1;32m 176\u001b[0m serialize_object(kwargs[key], buffer_threshold, item_threshold)\n\u001b[1;32m 177\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m kw_keys\n\u001b[1;32m 178\u001b[0m )\n\u001b[1;32m 179\u001b[0m )\n\u001b[1;32m 181\u001b[0m info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(nargs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(args), narg_bufs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(arg_bufs), kw_keys\u001b[38;5;241m=\u001b[39mkw_keys)\n\u001b[0;32m--> 182\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[43mserialize_object\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 183\u001b[0m msg\u001b[38;5;241m.\u001b[39mappend(pickle\u001b[38;5;241m.\u001b[39mdumps(info, PICKLE_PROTOCOL))\n\u001b[1;32m 184\u001b[0m msg\u001b[38;5;241m.\u001b[39mextend(arg_bufs)\n", "File \u001b[0;32m~/dev/ip/parallel/ipyparallel/serialize/serialize.py:113\u001b[0m, in \u001b[0;36mserialize_object\u001b[0;34m(obj, buffer_threshold, item_threshold)\u001b[0m\n\u001b[1;32m 110\u001b[0m cobj \u001b[38;5;241m=\u001b[39m can(obj)\n\u001b[1;32m 111\u001b[0m buffers\u001b[38;5;241m.\u001b[39mextend(_extract_buffers(cobj, buffer_threshold))\n\u001b[0;32m--> 113\u001b[0m buffers\u001b[38;5;241m.\u001b[39minsert(\u001b[38;5;241m0\u001b[39m, \u001b[43mpickle\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdumps\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mPICKLE_PROTOCOL\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m buffers\n", "\u001b[0;31mTypeError\u001b[0m: cannot pickle '_io.TextIOWrapper' object" ] } ], "source": [ "view.apply_sync(closed, 3)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Oops, no dice. For IPython to work with dill,\n", "there are one or two more steps. IPython will do these for you if you call `DirectView.use_dill`:" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "rc[:].use_dill()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now let's try again" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "15" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "view.apply_sync(closed, 3)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "25810: 10\n", "26717: 15\n" ] } ], "source": [ "!cat /tmp/dilltest" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Yay! Now we can use dill to allow ipyparallel to send anything.\n", "\n", "And that's it! We can send closures, open file handles, and other previously non-pickleables to our engines.\n", "\n", "Let's give it a try now:" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "20" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "remote_closure = view.apply_sync(make_closure, 4)\n", "remote_closure(5)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "25810: 10\n", "26717: 15\n", "25810: 20\n" ] } ], "source": [ "!cat /tmp/dilltest" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "But wait, there's more!\n", "\n", "At this point, we can send/recv all kinds of stuff" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "def outer(a):\n", " def inner(b):\n", " def inner_again(c):\n", " return c * b * a\n", "\n", " return inner_again\n", "\n", " return inner" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "So outer returns a function with a closure, which returns a function with a closure.\n", "\n", "Now, we can resolve the first closure on the engine, the second here, and the third on a different engine,\n", "after passing through a lambda we define here and call there, just for good measure." ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "6" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "view.apply_sync(lambda f: f(3), view.apply_sync(outer, 1)(2))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And for good measure, let's test that normal execution still works:" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "%%px\n", "foo = 5" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[5, 5]\n" ] }, { "data": { "text/plain": [ "[10, 10]" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "print(rc[:]['foo'])\n", "rc[:]['bar'] = lambda: 2 * foo\n", "rc[:].apply_sync(ipp.Reference('bar'))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And test that the `@interactive` decorator works" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Writing testdill.py\n" ] } ], "source": [ "%%file testdill.py\n", "import ipyparallel as ipp\n", "\n", "@ipp.interactive\n", "class C:\n", " a = 5\n", "\n", "@ipp.interactive\n", "class D(C):\n", " b = 10\n", "\n", "@ipp.interactive\n", "def foo(a):\n", " return a * b\n" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "import testdill" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5 10\n" ] } ], "source": [ "v = rc[-1]\n", "v['D'] = testdill.D\n", "d = v.apply_sync(lambda: D()) # noqa: F821\n", "print(d.a, d.b)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "50" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "v['b'] = 10\n", "v.apply_sync(testdill.foo, 5)" ] } ], "metadata": { "gist_id": "5241793", "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.10" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": {}, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 4 }