More documentation.

src/boot/boot.janet

@@ -2173,10 +2173,7 @@
   (defn expand-bindings [x]
     (case (type x)
       :array (map expand-bindings x)
-      :tuple (let [ctor (if (= :brackets (tuple/type x))
-                          tuple/brackets
-                          tuple)]
-               (ctor ;(map expand-bindings x)))
+      :tuple (keep-syntax! x (map expand-bindings x))
       :table (dotable x expand-bindings)
       :struct (table/to-struct (dotable x expand-bindings))
       (recur x)))
@@ -2184,7 +2181,7 @@
   (defn expanddef [t]
     (def last (in t (- (length t) 1)))
     (def bound (in t 1))
-    (tuple/slice
+    (keep-syntax! t
       (array/concat
         @[(in t 0) (expand-bindings bound)]
         (tuple/slice t 2 -2)
@@ -2199,10 +2196,10 @@
     (if (symbol? t1)
       (do
         (def args (map recur (tuple/slice t 3)))
-        (tuple 'fn t1 (in t 2) ;args))
+        (keep-syntax t (tuple 'fn t1 (in t 2) ;args)))
       (do
         (def args (map recur (tuple/slice t 2)))
-        (tuple 'fn t1 ;args))))
+        (keep-syntax t (tuple 'fn t1 ;args)))))
 
   (defn expandqq [t]
     (defn qq [x]

src/include/janet.h

@@ -1138,6 +1138,7 @@ struct JanetFunction {
     JanetFuncEnv *envs[];
 };
 
+/* Use to read Janet data structures into memory from source code */
 typedef struct JanetParseState JanetParseState;
 typedef struct JanetParser JanetParser;
 
@@ -1187,7 +1188,10 @@ typedef struct {
     const JanetAbstractType *at;
 } JanetMarshalContext;
 
-/* Defines an abstract type */
+/* Defines an abstract type. Use a const pointer to one of these structures
+ * when creating abstract types. The memory for this pointer should not be free
+ * until after janet_deinit is called. Usually, this means declaring JanetAbstractType's
+ * as const data at file scope, and creating instances with janet_abstract(&MyType, sizeof(MyTypeStruct)); */
 struct JanetAbstractType {
     const char *name;
     int (*gc)(void *data, size_t len);
@@ -1479,10 +1483,16 @@ JANET_API void janet_ev_dec_refcount(void);
 JANET_API void *janet_abstract_begin_threaded(const JanetAbstractType *atype, size_t size);
 JANET_API void *janet_abstract_end_threaded(void *x);
 JANET_API void *janet_abstract_threaded(const JanetAbstractType *atype, size_t size);
+
+/* Allow reference counting on threaded abstract types. This is useful when external code , either
+ * in the current OS thread or in a different OS thread, takes a pointer to this abstract type. The programmer
+ * should tncrement the reference count when taking the pointer, and then decrement and possibly cleanup and free
+ * if the reference count is 0. */
 JANET_API int32_t janet_abstract_incref(void *abst);
 JANET_API int32_t janet_abstract_decref(void *abst);
 
-/* If this returns 0, *abst will be deinitialized and freed */
+/* If this returns 0, *abst will be deinitialized and freed. Useful shorthand if there is no other cleanup for
+ * this abstract type before calling `janet_free` on it's backing memory. */
 JANET_API int32_t janet_abstract_decref_maybe_free(void *abst);
 
 /* Expose channel utilities */
@@ -1493,7 +1503,7 @@ JANET_API JanetChannel *janet_optchannel(const Janet *argv, int32_t argc, int32_
 JANET_API int janet_channel_give(JanetChannel *channel, Janet x);
 JANET_API int janet_channel_take(JanetChannel *channel, Janet *out);
 
-/* Expose some OS sync primitives */
+/* Expose some OS sync primitives - mutexes and reader-writer locks */
 JANET_API size_t janet_os_mutex_size(void);
 JANET_API size_t janet_os_rwlock_size(void);
 JANET_API void janet_os_mutex_init(JanetOSMutex *mutex);
@@ -1561,7 +1571,8 @@ JANET_API void janet_ev_post_event(JanetVM *vm, JanetCallback cb, JanetEVGeneric
 /* Callback used by janet_ev_threaded_await */
 JANET_API void janet_ev_default_threaded_callback(JanetEVGenericMessage return_value);
 
-/* Read async from a stream */
+/* Read async from a stream. These function yield to the event-loop with janet_await(), and so do not return.
+ * When the fiber is resumed, the fiber will simply continue to the next Janet abstract machine instruction. */
 JANET_NO_RETURN JANET_API void janet_ev_read(JanetStream *stream, JanetBuffer *buf, int32_t nbytes);
 JANET_NO_RETURN JANET_API void janet_ev_readchunk(JanetStream *stream, JanetBuffer *buf, int32_t nbytes);
 #ifdef JANET_NET
@@ -1570,7 +1581,8 @@ JANET_NO_RETURN JANET_API void janet_ev_recvchunk(JanetStream *stream, JanetBuff
 JANET_NO_RETURN JANET_API void janet_ev_recvfrom(JanetStream *stream, JanetBuffer *buf, int32_t nbytes, int flags);
 #endif
 
-/* Write async to a stream */
+/* Write async to a stream. These function yield to the event-loop with janet_await(), and so do not return.
+ * When the fiber is resumed, the fiber will simply continue to the next Janet abstract machine instruction. */
 JANET_NO_RETURN JANET_API void janet_ev_write_buffer(JanetStream *stream, JanetBuffer *buf);
 JANET_NO_RETURN JANET_API void janet_ev_write_string(JanetStream *stream, JanetString str);
 #ifdef JANET_NET
@@ -1582,17 +1594,63 @@ JANET_NO_RETURN JANET_API void janet_ev_sendto_string(JanetStream *stream, Janet
 
 #endif
 
-/* Parsing */
+/* Parsing.
+ *
+ * E.g.
+ *
+ * JanetParser parser;
+ * janet_parser_init(&parser);
+ * for (int i = 0; i < source_code_length + 1; i++) {
+ *   if (i >= source_code_length) {
+ *     janet_parser_eof(&parser);
+ *   } else {
+ *     janet_parser_consume(&parser, source_code[i]);
+ *   }
+ *   while (janet_parser_has_more(&parser)) {
+ *      Janet x = janet_parser_produce(&parser);
+ *      janet_printf("got value: %v\n", x);
+ *   }
+ *   switch (janet_parser_status(&parser)) {
+ *      case JANET_PARSE_PENDING: break;
+ *      case JANET_PARSE_ERROR: janet_eprintf("error: %s\n", janet_parser_error(&parser)); break;
+ *      case JANET_PARSE_ROOT: break;
+ *      case JANET_PARSE_DEAD: break;
+ *   }
+ * }
+ * janet_parser_deinit(&parser);
+ *
+ * */
 extern JANET_API const JanetAbstractType janet_parser_type;
+
+/* Construct/destruct a parser. Parsers can be allocated on the stack or the heap. */
 JANET_API void janet_parser_init(JanetParser *parser);
 JANET_API void janet_parser_deinit(JanetParser *parser);
+
+/* Feed bytes into the parser. Check the parser state after every byte to handle errors. */
 JANET_API void janet_parser_consume(JanetParser *parser, uint8_t c);
+
+/* Check the current status of the parser */
 JANET_API enum JanetParserStatus janet_parser_status(JanetParser *parser);
+
+/* Produce a value from the parser. Call this when janet_parser_has_more(&parser) is non-zero. */
 JANET_API Janet janet_parser_produce(JanetParser *parser);
+
+/* Produce a value from the parser, wrapped in a tuple. The tuple is used to carry the source mapping information of the
+ * top level form, such as a line number or symbol. */
 JANET_API Janet janet_parser_produce_wrapped(JanetParser *parser);
+
+/* When there is an error while parsing (janet_parser_status(&parser) == JANET_PARSE_ERROR), get a nice error string.
+ * Calling this will also flush the parser. */
 JANET_API const char *janet_parser_error(JanetParser *parser);
+
+/* If there is a parsing error, flush the parser to set the state back to empty.
+ * This allows for better error recover and less confusing error messages on bad syntax deep inside nested data structures. */
 JANET_API void janet_parser_flush(JanetParser *parser);
+
+/* Indicate that there is no more source code */
 JANET_API void janet_parser_eof(JanetParser *parser);
+
+/* If non-zero, the parser has values ready to be produced. */
 JANET_API int janet_parser_has_more(JanetParser *parser);
 
 /* Assembly */
@@ -1636,7 +1694,10 @@ JANET_API JanetCompileResult janet_compile_lint(
 JANET_API JanetTable *janet_core_env(JanetTable *replacements);
 JANET_API JanetTable *janet_core_lookup_table(JanetTable *replacements);
 
-/* Execute strings */
+/* Execute strings.
+ *
+ * These functions wrap parsing, compilation, and evalutation into convenient functions.
+ * */
 #define JANET_DO_ERROR_RUNTIME 0x01
 #define JANET_DO_ERROR_COMPILE 0x02
 #define JANET_DO_ERROR_PARSE 0x04
@@ -1830,21 +1891,41 @@ JANET_API JanetTable *janet_env_lookup(JanetTable *env);
 JANET_API void janet_env_lookup_into(JanetTable *renv, JanetTable *env, const char *prefix, int recurse);
 
 /* GC */
-JANET_API void janet_mark(Janet x);
-JANET_API void janet_sweep(void);
+
+/* The main interface to garbage collection. Call this to do a full mark and sweep cleanup. */
 JANET_API void janet_collect(void);
-JANET_API void janet_clear_memory(void);
+
+/* Add "roots" to the garbage collector to prevent the runtime from freeing objects.
+ * This is only needed if code outside of Janet keeps references to Janet values */
 JANET_API void janet_gcroot(Janet root);
 JANET_API int janet_gcunroot(Janet root);
-JANET_API int janet_gcunrootall(Janet root);
+
+/* Allow disabling garbage collection temporarily or for certain sections of code.
+ * this is a very cheap operation. */
 JANET_API int janet_gclock(void);
 JANET_API void janet_gcunlock(int handle);
+
+/* The mark and sweep components of the mark and sweep collector. Prefer using janet_collect directly. */
+JANET_API void janet_mark(Janet x);
+JANET_API void janet_sweep(void);
+
+/* Clear all gced memory and call all destructors. Used as part of the standard cleanup routune, most programmers will not need this. */
+JANET_API void janet_clear_memory(void);
+
+/* Remove all GC roots. Used as part of the standard cleanup routine, most programmers will not need this. */
+JANET_API int janet_gcunrootall(Janet root);
+
+/* Hint to the collector that memory of size s was just allocated to help it better understand when to free memory. */
 JANET_API void janet_gcpressure(size_t s);
 
 /* Functions */
 JANET_API JanetFuncDef *janet_funcdef_alloc(void);
 JANET_API JanetFunction *janet_thunk(JanetFuncDef *def);
+
+/* Get a function that when called with no args, will return x. */
 JANET_API JanetFunction *janet_thunk_delay(Janet x);
+
+/* Do some simple verfification on constructed bytecode to disallow any trivial incorrect bytecode. */
 JANET_API int janet_verify(JanetFuncDef *def);
 
 /* Pretty printing */
@@ -1966,7 +2047,14 @@ JANET_API JanetBinding janet_resolve_ext(JanetTable *env, JanetSymbol sym);
 /* Get values from the core environment. */
 JANET_API Janet janet_resolve_core(const char *name);
 
-/* New C API */
+/* New C API
+ *
+ * The "New" C API is intended to make constructing good documentation and source maps
+ * much more straightforward. This not only ensures doc strings for functions in native
+ * modules, it also add source code mapping for C functions so that programmers can see which
+ * file and line a native function that calls janet_panic came from.
+ *
+ * */
 
 /* Shorthand for janet C function declarations */
 #define JANET_CFUN(name) Janet name (int32_t argc, Janet *argv)