fixes #1626

Author: jph00

nbdev/_modidx.py

@@ -385,5 +385,6 @@
                             'nbdev.sync.absolute_import': ('api/sync.html#absolute_import', 'nbdev/sync.py'),
                             'nbdev.sync.nbdev_update': ('api/sync.html#nbdev_update', 'nbdev/sync.py')},
             'nbdev.test': { 'nbdev.test._keep_file': ('api/test.html#_keep_file', 'nbdev/test.py'),
+                            'nbdev.test._test_nb_sync': ('api/test.html#_test_nb_sync', 'nbdev/test.py'),
                             'nbdev.test.nbdev_test': ('api/test.html#nbdev_test', 'nbdev/test.py'),
                             'nbdev.test.test_nb': ('api/test.html#test_nb', 'nbdev/test.py')}}}

nbdev/processors.py

@@ -12,6 +12,8 @@
 # %% ../nbs/api/10_processors.ipynb #2398f5ef-06d3-4890-8a54-7cf4f81f3894
 import ast
 
+from fastcore.aio import run_sync
+
 from .config import *
 from .imports import *
 from .process import *
@@ -255,7 +257,7 @@ def __call__(self, cell):
         if not self.nb.has_docs_ or not hasattr(self, 'k'): return
         fm = getattr(self.nb, 'frontmatter_', {})
         if str2bool(fm.get('skip_showdoc', False)): return
-        if _do_eval(cell): self.k.cell(cell)
+        if _do_eval(cell): run_sync(self.k.cell(cell))
         title = fm.get('title', '')
         if self.k.exc: 
             raise Exception(f"Error{' in notebook: '+title if title else ''} in cell {cell.idx_} :\n{cell.source}") from self.k.exc

nbdev/test.py

@@ -8,7 +8,7 @@
 __all__ = ['test_nb', 'nbdev_test']
 
 # %% ../nbs/api/12_test.ipynb #45e10c3f
-import time,os,sys,traceback,contextlib,inspect,faulthandler,signal
+import time,os,sys,traceback,contextlib,inspect,faulthandler,signal,asyncio
 from fastcore.basics import *
 from fastcore.imports import *
 from fastcore.foundation import *
@@ -25,7 +25,7 @@
 from execnb.shell import *
 
 # %% ../nbs/api/12_test.ipynb #3f4fa1ad
-def test_nb(
+async def test_nb(
     fn,  # file name of notebook to test
     skip_flags=None,  # list of flags marking cells to skip
     force_flags=None,  # list of flags marking cells to always run
@@ -61,7 +61,7 @@ def _no_eval(cell):
         if do_print: print(f'Starting {fn}')
         try:
             with working_directory(fn.parent):
-                k.run_all(nb, exc_stop=True, preproc=_no_eval, verbose=verbose)
+                await k.run_all(nb, exc_stop=True, preproc=_no_eval, verbose=verbose)
                 if save: write_nb(nb, fn)
                 res = True
         except: 
@@ -74,6 +74,9 @@ def _no_eval(cell):
         else: os.environ['IN_TEST'] = prev_test
 
 
+# %% ../nbs/api/12_test.ipynb #12516e0a
+def _test_nb_sync(fn, **kwargs): return asyncio.run(test_nb(fn, **kwargs))
+
 # %% ../nbs/api/12_test.ipynb #d8bf1f1b-935d-4b69-ba96-827c5d7213f0
 def _keep_file(
     p:Path, # filename for which to check for `indicator_fname`
@@ -112,7 +115,7 @@ def nbdev_test(
     else: kw = {'method':'spawn'} if sys.platform=='darwin' else {}
     wd_pth = cfg.nbs_path
     with working_directory(wd_pth if (wd_pth and wd_pth.exists()) else os.getcwd()):
-        results = parallel(test_nb, files, skip_flags=skip_flags, force_flags=force_flags, n_workers=n_workers,
+        results = parallel(_test_nb_sync, files, skip_flags=skip_flags, force_flags=force_flags, n_workers=n_workers,
                            basepath=cfg.config_path, pause=pause, do_print=do_print, verbose=verbose, save=save, **kw)
     passed,times = zip(*results)
     if all(passed): print("Success.")

nbs/api/10_processors.ipynb

@@ -30,6 +30,8 @@
     "#| export\n",
     "import ast\n",
     "\n",
+    "from fastcore.aio import run_sync\n",
+    "\n",
     "from nbdev.config import *\n",
     "from nbdev.imports import *\n",
     "from nbdev.process import *\n",
@@ -763,7 +765,7 @@
     "        if not self.nb.has_docs_ or not hasattr(self, 'k'): return\n",
     "        fm = getattr(self.nb, 'frontmatter_', {})\n",
     "        if str2bool(fm.get('skip_showdoc', False)): return\n",
-    "        if _do_eval(cell): self.k.cell(cell)\n",
+    "        if _do_eval(cell): run_sync(self.k.cell(cell))\n",
     "        title = fm.get('title', '')\n",
     "        if self.k.exc: \n",
     "            raise Exception(f\"Error{' in notebook: '+title if title else ''} in cell {cell.idx_} :\\n{cell.source}\") from self.k.exc\n",

nbs/api/12_test.ipynb

@@ -28,7 +28,7 @@
    "outputs": [],
    "source": [
     "#| export\n",
-    "import time,os,sys,traceback,contextlib,inspect,faulthandler,signal\n",
+    "import time,os,sys,traceback,contextlib,inspect,faulthandler,signal,asyncio\n",
     "from fastcore.basics import *\n",
     "from fastcore.imports import *\n",
     "from fastcore.foundation import *\n",
@@ -53,7 +53,7 @@
    "outputs": [],
    "source": [
     "#| export\n",
-    "def test_nb(\n",
+    "async def test_nb(\n",
     "    fn,  # file name of notebook to test\n",
     "    skip_flags=None,  # list of flags marking cells to skip\n",
     "    force_flags=None,  # list of flags marking cells to always run\n",
@@ -89,7 +89,7 @@
     "        if do_print: print(f'Starting {fn}')\n",
     "        try:\n",
     "            with working_directory(fn.parent):\n",
-    "                k.run_all(nb, exc_stop=True, preproc=_no_eval, verbose=verbose)\n",
+    "                await k.run_all(nb, exc_stop=True, preproc=_no_eval, verbose=verbose)\n",
     "                if save: write_nb(nb, fn)\n",
     "                res = True\n",
     "        except: \n",
@@ -102,6 +102,25 @@
     "        else: os.environ['IN_TEST'] = prev_test\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "d7a14cc3",
+   "metadata": {},
+   "source": [
+    "`test_nb` is async. `parallel` workers are fresh loopless processes, so each enters through `asyncio.run` at its process boundary, which keeps cells on the worker's main thread (where SIGALRM timeouts can arm). `run_sync` is reserved for sync callers already inside a running loop, like `exec_show_docs` under the sync processor pipeline.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12516e0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def _test_nb_sync(fn, **kwargs): return asyncio.run(test_nb(fn, **kwargs))"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "9dc26b80-6e4e-4a16-bcde-dd4d156289e5",
@@ -118,7 +137,7 @@
    "outputs": [],
    "source": [
     "_nb = Path('../../tests/directives.ipynb')\n",
-    "success,duration = test_nb(_nb, skip_flags=['notest'])\n",
+    "success,duration = await test_nb(_nb, skip_flags=['notest'])\n",
     "assert success"
    ]
   },
@@ -138,7 +157,7 @@
    "outputs": [],
    "source": [
     "_nb = Path('../../tests/directives.ipynb')\n",
-    "success,duration = test_nb(_nb, showerr=False)\n",
+    "success,duration = await test_nb(_nb, showerr=False)\n",
     "assert not success"
    ]
   },
@@ -175,8 +194,8 @@
     "    nbp = td/'prof.ipynb'\n",
     "    write_nb(new_nb([mk_cell('assert prof_x==7')]), nbp)\n",
     "    with modified_env(IPYTHONDIR=str(td)):\n",
-    "        assert     test_nb(nbp, showerr=False)[0]\n",
-    "        assert not test_nb(nbp, showerr=False, profile=False)[0]"
+    "        assert     (await test_nb(nbp, showerr=False))[0]\n",
+    "        assert not (await test_nb(nbp, showerr=False, profile=False))[0]"
    ]
   },
   {
@@ -240,7 +259,7 @@
     "    else: kw = {'method':'spawn'} if sys.platform=='darwin' else {}\n",
     "    wd_pth = cfg.nbs_path\n",
     "    with working_directory(wd_pth if (wd_pth and wd_pth.exists()) else os.getcwd()):\n",
-    "        results = parallel(test_nb, files, skip_flags=skip_flags, force_flags=force_flags, n_workers=n_workers,\n",
+    "        results = parallel(_test_nb_sync, files, skip_flags=skip_flags, force_flags=force_flags, n_workers=n_workers,\n",
     "                           basepath=cfg.config_path, pause=pause, do_print=do_print, verbose=verbose, save=save, **kw)\n",
     "    passed,times = zip(*results)\n",
     "    if all(passed): print(\"Success.\")\n",

pyproject.toml

@@ -19,7 +19,7 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
 ]
 dynamic = ["version"]
-dependencies = [ "fastcore>=2.2.3", "execnb>=0.2.8", "astunparse", "ghapi>=2.0.2", "watchdog", "asttokens",
+dependencies = [ "fastcore>=2.2.3", "execnb>=0.2.11", "astunparse", "ghapi>=2.0.2", "watchdog", "asttokens",
     "setuptools", "build", "fastgit>=0.0.7", "pyyaml", "tomli; python_version < '3.11'", ]
 
 [project.optional-dependencies]