initial draft of fix for issue #2498:

1. make /// ... and //! ... and /** ... */ and /*! ... */ into sugar for #[doc = ...] attributes.
2. add a script in etc/ to help converting doc-attributes to doc-comments
3. add some functions to core::str to help with (1)

Author: Dretch

src/etc/sugarise-doc-comments.py

@@ -0,0 +1,82 @@
+#!/usr/bin/python
+
+#
+# this script attempts to turn doc comment attributes (#[doc = "..."])
+# into sugared-doc-comments (/** ... */ and /// ...)
+#
+# it sugarises all .rs/.rc files underneath the working directory
+#
+
+import sys, os, fnmatch, re
+
+
+DOC_PATTERN = '^(?P<indent>[\\t ]*)#\\[(\\s*)doc(\\s*)=' + \
+              '(\\s*)"(?P<text>(\\"|[^"])*?)"(\\s*)\\]' + \
+              '(?P<semi>;)?'
+
+ESCAPES = [("\\'", "'"),
+           ('\\"', '"'),
+           ("\\n", "\n"),
+           ("\\r", "\r"),
+           ("\\t", "\t")]
+
+
+def unescape(s):
+    for (find, repl) in ESCAPES:
+        s = s.replace(find, repl)
+    return s
+
+
+def block_trim(s):
+    lns = s.splitlines()
+
+    # remove leading/trailing whitespace-lines
+    while lns and not lns[0].strip():
+        lns = lns[1:]
+    while lns and not lns[-1].strip():
+        lns = lns[:-1]
+
+    # remove leading horizontal whitespace
+    n = sys.maxint
+    for ln in lns:
+        if ln.strip():
+            n = min(n, len(re.search('^\s*', ln).group()))
+    if n != sys.maxint:
+        lns = [ln[n:] for ln in lns]
+
+    # strip trailing whitespace
+    lns = [ln.rstrip() for ln in lns]
+
+    return lns
+
+
+def replace_doc(m):
+    indent = m.group('indent')
+    text = block_trim(unescape(m.group('text')))
+
+    if len(text) > 1:
+        inner = '!' if m.group('semi') else '*'
+        starify = lambda s: indent + ' *' + (' ' + s if s else '')
+        text = '\n'.join(map(starify, text))
+        repl = indent + '/*' + inner + '\n' + text + '\n' + indent + ' */'
+    else:
+        inner = '!' if m.group('semi') else '/'
+        repl = indent + '//' + inner + ' ' + text[0]
+
+    return repl
+
+
+def sugarise_file(path):
+    s = open(path).read()
+
+    r = re.compile(DOC_PATTERN, re.MULTILINE | re.DOTALL)
+    ns = re.sub(r, replace_doc, s)
+
+    if s != ns:
+        open(path, 'w').write(ns)
+
+
+for (dirpath, dirnames, filenames) in os.walk('.'):
+    for name in fnmatch.filter(filenames, '*.r[sc]'):
+        sugarise_file(os.path.join(dirpath, name))
+

src/libcore/str.rs

@@ -58,8 +58,8 @@ export
    all, any,
    all_between, any_between,
    map,
-   each,
-   each_char,
+   each, eachi,
+   each_char, each_chari,
    bytes_iter,
    chars_iter,
    split_char_iter,
@@ -73,7 +73,7 @@ export
    find_char, find_char_from, find_char_between,
    rfind_char, rfind_char_from, rfind_char_between,
    find_str, find_str_from, find_str_between,
-   contains,
+   contains, contains_char,
    starts_with,
    ends_with,
 
@@ -672,22 +672,35 @@ pure fn bytes_iter(ss: str/&, it: fn(u8)) {
 #[doc = "Iterate over the bytes in a string"]
 #[inline(always)]
 pure fn each(s: str/&, it: fn(u8) -> bool) {
+    eachi(s, {|_i, b| it(b)})
+}
+
+#[doc = "Iterate over the bytes in a string, with indices"]
+#[inline(always)]
+pure fn eachi(s: str/&, it: fn(uint, u8) -> bool) {
     let mut i = 0u, l = len(s);
     while (i < l) {
-        if !it(s[i]) { break; }
+        if !it(i, s[i]) { break; }
         i += 1u;
     }
 }
 
 #[doc = "Iterates over the chars in a string"]
 #[inline(always)]
 pure fn each_char(s: str/&, it: fn(char) -> bool) {
-    let mut pos = 0u;
+    each_chari(s, {|_i, c| it(c)})
+}
+
+#[doc = "Iterates over the chars in a string, with indices"]
+#[inline(always)]
+pure fn each_chari(s: str/&, it: fn(uint, char) -> bool) {
+    let mut pos = 0u, ch_pos = 0u;
     let len = len(s);
     while pos < len {
         let {ch, next} = char_range_at(s, pos);
         pos = next;
-        if !it(ch) { break; }
+        if !it(ch_pos, ch) { break; }
+        ch_pos += 1u;
     }
 }
 
@@ -1146,6 +1159,18 @@ pure fn contains(haystack: str/&a, needle: str/&b) -> bool {
     option::is_some(find_str(haystack, needle))
 }
 
+#[doc = "
+Returns true if a string contains a char.
+
+# Arguments
+
+* haystack - The string to look in
+* needle - The char to look for
+"]
+pure fn contains_char(haystack: str/&, needle: char) -> bool {
+    option::is_some(find_char(haystack, needle))
+}
+
 #[doc = "
 Returns true if one string starts with another
 
@@ -1879,12 +1904,21 @@ impl extensions/& for str/& {
     #[doc = "Returns true if one string contains another"]
     #[inline]
     fn contains(needle: str/&a) -> bool { contains(self, needle) }
+    #[doc = "Returns true if a string contains a char"]
+    #[inline]
+    fn contains_char(needle: char) -> bool { contains_char(self, needle) }
     #[doc = "Iterate over the bytes in a string"]
     #[inline]
     fn each(it: fn(u8) -> bool) { each(self, it) }
+    #[doc = "Iterate over the bytes in a string, with indices"]
+    #[inline]
+    fn eachi(it: fn(uint, u8) -> bool) { eachi(self, it) }
     #[doc = "Iterate over the chars in a string"]
     #[inline]
     fn each_char(it: fn(char) -> bool) { each_char(self, it) }
+    #[doc = "Iterate over the chars in a string, with indices"]
+    #[inline]
+    fn each_chari(it: fn(uint, char) -> bool) { each_chari(self, it) }
     #[doc = "Returns true if one string ends with another"]
     #[inline]
     fn ends_with(needle: str/&) -> bool { ends_with(self, needle) }
@@ -2644,6 +2678,14 @@ mod tests {
         assert !contains(data, "ไท华");
     }
 
+    #[test]
+    fn test_contains_char() {
+        assert contains_char("abc", 'b');
+        assert contains_char("a", 'a');
+        assert !contains_char("abc", 'd');
+        assert !contains_char("", 'a');
+    }
+
     #[test]
     fn test_chars_iter() {
         let mut i = 0;

src/libsyntax/ast.rs

@@ -652,8 +652,9 @@ type attribute = spanned<attribute_>;
 #[auto_serialize]
 enum attr_style { attr_outer, attr_inner, }
 
+// doc-comments are promoted to attributes that have is_sugared_doc = true
 #[auto_serialize]
-type attribute_ = {style: attr_style, value: meta_item};
+type attribute_ = {style: attr_style, value: meta_item, is_sugared_doc: bool};
 
 /*
   iface_refs appear in both impls and in classes that implement ifaces.

src/libsyntax/attr.rs

@@ -4,18 +4,21 @@ import std::map;
 import std::map::hashmap;
 import either::either;
 import diagnostic::span_handler;
-import ast_util::dummy_spanned;
+import ast_util::{spanned, dummy_spanned};
+import parse::comments::{doc_comment_style, strip_doc_comment_decoration};
 
 // Constructors
 export mk_name_value_item_str;
 export mk_name_value_item;
 export mk_list_item;
 export mk_word_item;
 export mk_attr;
+export mk_sugared_doc_attr;
 
 // Conversion
 export attr_meta;
 export attr_metas;
+export desugar_doc_attr;
 
 // Accessors
 export get_attr_name;
@@ -66,9 +69,19 @@ fn mk_word_item(+name: ast::ident) -> @ast::meta_item {
 }
 
 fn mk_attr(item: @ast::meta_item) -> ast::attribute {
-    ret dummy_spanned({style: ast::attr_inner, value: *item});
+    ret dummy_spanned({style: ast::attr_inner, value: *item,
+                       is_sugared_doc: false});
 }
 
+fn mk_sugared_doc_attr(text: str, lo: uint, hi: uint) -> ast::attribute {
+    let lit = spanned(lo, hi, ast::lit_str(@text));
+    let attr = {
+        style: doc_comment_style(text),
+        value: spanned(lo, hi, ast::meta_name_value(@"doc", lit)),
+        is_sugared_doc: true
+    };
+    ret spanned(lo, hi, attr);
+}
 
 /* Conversion */
 
@@ -81,6 +94,16 @@ fn attr_metas(attrs: [ast::attribute]/~) -> [@ast::meta_item]/~ {
     ret mitems;
 }
 
+fn desugar_doc_attr(attr: ast::attribute) -> ast::attribute {
+    if attr.node.is_sugared_doc {
+        let comment = get_meta_item_value_str(@attr.node.value).get();
+        let meta = mk_name_value_item_str(@"doc",
+                                     strip_doc_comment_decoration(*comment));
+        ret mk_attr(meta);
+    } else {
+        attr
+    }
+}
 
 /* Accessors */
 

src/libsyntax/fold.rs

@@ -102,7 +102,8 @@ fn fold_meta_item_(&&mi: @meta_item, fld: ast_fold) -> @meta_item {
 fn fold_attribute_(at: attribute, fld: ast_fold) ->
    attribute {
     ret {node: {style: at.node.style,
-                value: *fold_meta_item_(@at.node.value, fld)},
+                value: *fold_meta_item_(@at.node.value, fld),
+                is_sugared_doc: at.node.is_sugared_doc },
          span: fld.new_span(at.span)};
 }
 //used in noop_fold_foreign_item and noop_fold_fn_decl