#############################################################################
##
#W  Text.gi                      GAPDoc                          Frank Lübeck
##
#H  @(#)$Id: Text.gi,v 1.15 2007/10/29 11:18:37 gap Exp $
##
#Y  Copyright (C)  2000,  Frank Lübeck,  Lehrstuhl D für Mathematik,  
#Y  RWTH Aachen
##
##  The files Text.g{d,i}  contain some utilities for  dealing with text
##  strings.
##  

##  
##  <#GAPDoc Label="CharsColls">
##  <ManSection>
##  <Var Name="WHITESPACE" />
##  <Var Name="CAPITALLETTERS" />
##  <Var Name="SMALLLETTERS" />
##  <Var Name="LETTERS" />
##  <Var Name="DIGITS" />
##  <Var Name="HEXDIGITS" />
##  <Description>
##  These variables contain sets of characters which are useful for
##  text processing. They are defined as follows.<P/>
##  <List >
##  <Mark><C>WHITESPACE</C></Mark>
##  <Item><C>" \n\t\r"</C></Item>
##  <Mark><C>CAPITALLETTERS</C></Mark>
##  <Item><C>"ABCDEFGHIJKLMNOPQRSTUVWXYZ"</C></Item>
##  <Mark><C>SMALLLETTERS</C></Mark>
##  <Item><C>"abcdefghijklmnopqrstuvwxyz"</C></Item>
##  <Mark><C>LETTERS</C></Mark>
##  <Item>concatenation of <C>CAPITALLETTERS</C> and <C>SMALLLETTERS</C></Item>
##  <Mark><C>DIGITS</C></Mark><Item><C>"0123456789"</C></Item>
##  <Mark><C>HEXDIGITS</C></Mark><Item><C>"0123456789ABCDEFabcdef"</C></Item>
##  </List>
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
##  
InstallValue(WHITESPACE, " \n\t\r");
InstallValue(CAPITALLETTERS, "ABCDEFGHIJKLMNOPQRSTUVWXYZ");
IsSet(CAPITALLETTERS);
InstallValue(SMALLLETTERS, "abcdefghijklmnopqrstuvwxyz");
IsSet(SMALLLETTERS);
InstallValue(LETTERS, Concatenation(CAPITALLETTERS, SMALLLETTERS));
IsSet(LETTERS);
InstallValue(DIGITS, "0123456789");
InstallValue(HEXDIGITS, "0123456789ABCDEFabcdef");


##  
##  <#GAPDoc Label="TextAttr">
##  <ManSection>
##  <Var Name="TextAttr" />
##  <Description>
##  The  record  <Ref Var="TextAttr"/>  contains  strings  which can  be
##  printed  to   change  the  terminal  attribute   for  the  following
##  characters. This  only works  with terminals which  understand basic
##  ANSI escape sequences.  Try the following example to see  if this is
##  the case for the terminal you are  using. It shows the effect of the
##  foreground and background color  attributes and of the <C>.bold</C>,
##  <C>.blink</C>, <C>.normal</C>, <C>.reverse</C> and<C>.underscore</C>
##  which can partly be mixed.
##  
##  <Listing Type="Example">
##  extra := ["CSI", "reset", "delline", "home"];;
##  for t in Difference(RecNames(TextAttr), extra) do
##    Print(TextAttr.(t), "TextAttr.", t, TextAttr.reset,"\n");
##  od;
##  </Listing>
##  
##  The suggested defaults for colors <C>0..7</C> are black, red, green,
##  brown, blue,  magenta, cyan,  white. But this  may be  different for
##  your terminal configuration.<P/>
##  
##  The  escape  sequence <C>.delline</C>  deletes  the  content of  the
##  current line and  <C>.home</C> moves the cursor to  the beginning of
##  the current line.
##  
##  <Listing Type="Example">
##  for i in [1..5] do 
##    Print(TextAttr.home, TextAttr.delline, String(i,-6), "\c"); 
##    Sleep(1); 
##  od;
##  </Listing>
##  
##  <Index>ANSI_COLORS</Index> 
##  Whenever  you  use  this  in   some  printing  routines  you  should
##  make  it optional.  Use  these attributes  only,  when the  variable
##  <C>ANSI_COLORS</C> has the value <K>true</K>.
##  </Description>
##  </ManSection>
##  
##  <#/GAPDoc>
##  
InstallValue(TextAttr, rec());
TextAttr.CSI := "\033[";
TextAttr.reset := Concatenation(TextAttr.CSI, "0m");
TextAttr.normal := Concatenation(TextAttr.CSI, "22m");
TextAttr.bold := Concatenation(TextAttr.CSI, "1m");
TextAttr.underscore := Concatenation(TextAttr.CSI, "4m");
TextAttr.blink := Concatenation(TextAttr.CSI, "5m");
TextAttr.reverse := Concatenation(TextAttr.CSI, "7m");
# foreground colors 0..7 (default: black, red, green, brown, blue, magenta,
# cyan, white
TextAttr.0 := Concatenation(TextAttr.CSI, "30m");
TextAttr.1 := Concatenation(TextAttr.CSI, "31m");
TextAttr.2 := Concatenation(TextAttr.CSI, "32m");
TextAttr.3 := Concatenation(TextAttr.CSI, "33m");
TextAttr.4 := Concatenation(TextAttr.CSI, "34m");
TextAttr.5 := Concatenation(TextAttr.CSI, "35m");
TextAttr.6 := Concatenation(TextAttr.CSI, "36m");
TextAttr.7 := Concatenation(TextAttr.CSI, "37m");
# background colors 0..7
TextAttr.b0 := Concatenation(TextAttr.CSI, "40m");
TextAttr.b1 := Concatenation(TextAttr.CSI, "41m");
TextAttr.b2 := Concatenation(TextAttr.CSI, "42m");
TextAttr.b3 := Concatenation(TextAttr.CSI, "43m");
TextAttr.b4 := Concatenation(TextAttr.CSI, "44m");
TextAttr.b5 := Concatenation(TextAttr.CSI, "45m");
TextAttr.b6 := Concatenation(TextAttr.CSI, "46m");
TextAttr.b7 := Concatenation(TextAttr.CSI, "47m");

TextAttr.delline := Concatenation(TextAttr.CSI, "2K");
TextAttr.home := Concatenation(TextAttr.CSI, "1G");



##  <#GAPDoc Label="RepeatedString">
##  <ManSection >
##  <Func Arg="c, len" Name="RepeatedString" />
##  <Description>
##  Here <A>c</A> must be either a  character or a string and <A>len</A>
##  is a non-negative number. Then <Ref Func="RepeatedString" /> returns
##  a string of length <A>len</A> consisting of copies of <A>c</A>.
##  <Example>
##  gap> RepeatedString('=',51);
##  "==================================================="
##  gap> RepeatedString("*=",51);
##  "*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*"
##  </Example>
##  </Description>
##  </ManSection>
##  
##  <#/GAPDoc>
InstallGlobalFunction(RepeatedString, function(s, n)
  local res, i;
  res := EmptyString(n+1);
  if n = 0 then
    return res;
  elif IsString(s) and Length(s) > 0 then
    if Length(s) > n then
      for i in [1..n] do
        res[i] := s[i];
      od;
      return s;
    else
      Append(res, s);
    fi;
  elif IsChar(s) then
    res[1] := s;
  else
    Error("First argument must be character or non-empty string\n");
  fi;
  while 2*Length(res) <= n do
    Append(res, res);
  od;
  Append(res, res{[1..n-Length(res)]});
  return res;
end);

##  <#GAPDoc Label="PositionMatchingDelimiter">
##  <ManSection >
##  <Func Arg="str, delim, pos" Name="PositionMatchingDelimiter" />
##  <Returns>position as integer or <K>fail</K></Returns>
##  <Description>
##  Here <A>str</A> must  be a string and <A>delim</A>  a string with
##  two  different characters.  This function  searches the  smallest
##  position   <C>r</C>  of   the  character   <C><A>delim</A>[2]</C>
##  in   <A>str</A>   such  that   the   number   of  occurrences  of
##  <C><A>delim</A>[2]</C>    in    <A>str</A>   between    positions
##  <C><A>pos</A>+1</C>  and  <C>r</C> is  by  one  greater than  the
##  corresponding number of occurrences of <C><A>delim</A>[1]</C>.<P/>
##  
##  If such an <C>r</C> exists, it is returned. Otherwise <K>fail</K>
##  is returned.
##  
##  <Example>
##  gap> PositionMatchingDelimiter("{}x{ab{c}d}", "{}", 0);
##  fail
##  gap> PositionMatchingDelimiter("{}x{ab{c}d}", "{}", 1);
##  2
##  gap> PositionMatchingDelimiter("{}x{ab{c}d}", "{}", 6);
##  11
##  </Example>
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
InstallGlobalFunction(PositionMatchingDelimiter, function(str, delim, pos)
  local   b,  e,  p,  l,  level;

  b := delim[1];
  e := delim[2];
  
  p := pos+1;
  l := Length(str);
  level := 0;
  while true do
    if p > l then
      return fail;
    elif str[p] = b then
      level := level+1;
    elif str[p] = e then
      if level = 0 then 
        return p;
      else
        level := level-1;
      fi;
    fi;
    p := p+1;
  od;
end);

##  <#GAPDoc Label="SubstitutionSublist">
##  <ManSection >
##  <Func Arg="list, sublist, new[, flag]" Name="SubstitutionSublist" />
##  <Returns>the changed list</Returns>
##  <Description>
##  This function looks for (non-overlapping) occurrences of a sublist
##  <A>sublist</A> in a list <A>list</A> (compare <Ref BookName="ref"
##  Oper="PositionSublist"  />) and  returns a  list where  these are
##  substituted with the list <A>new</A>.<P/>
##  
##  The  optional argument  <A>flag</A>  can  either be  <C>"all"</C>
##  (this is the default if not given) or <C>"one"</C>. In the second
##  case only  the first occurrence of <A>sublist</A> is substituted.
##  <P/>
##  
##  If <A>sublist</A> does not  occur in <A>list</A> then <A>list</A>
##  itself is returned (and not a <C>ShallowCopy(list)</C>).
##  
##  <Example>
##  gap> SubstitutionSublist("xababx", "ab", "a");
##  "xaax"
##  </Example>
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
InstallGlobalFunction(SubstitutionSublist, function(arg)
  local   str,  substr,  lss,  subs,  all,  p,  s, off;
  str := arg[1];
  substr := arg[2];
  lss := Length(substr);
  subs := arg[3];
  if Length(arg)>3 then
    all := arg[4]="all";
  else
    all := true;
  fi;
  
  p := PositionSublist(str, substr);
  if p = fail then 
    # return original object in case of no substitution
    return str; 
  fi;
  s := str{[]};
  off := 1-lss;
  while p<>fail do
    Append(s, str{[off+lss..p-1]});
    Append(s, subs);
##      str := str{[p+lss..Length(str)]};
    off := p;
    if all then
      p := PositionSublist(str, substr, p+lss-1);
    else
      p := fail;
    fi;
    if p=fail then
      Append(s, str{[off+lss..Length(str)]});
    fi;
  od;
  return s;
end);
    

##  <#GAPDoc Label="NumberDigits">
##  <ManSection >
##  <Func Arg="str, base" Name="NumberDigits" />
##  <Returns>integer</Returns>
##  <Func Arg="n, base" Name="DigitsNumber" />
##  <Returns>string</Returns>
##  <Description>
##  The argument  <A>str</A> of  <Ref Func="NumberDigits" />  must be
##  a  string  consisting  only  of an  optional  leading  <C>'-'</C>
##  and characters  in  <C>0123456789abcdefABCDEF</C>,  describing an
##  integer  in  base <A>base</A>  with  <M>2  \leq <A>base</A>  \leq
##  16</M>. This function returns the corresponding integer.<P/>
##  
##  The function <Ref Func="DigitsNumber" /> does the reverse.
##  
##  <Example>
##  gap> NumberDigits("1A3F",16);
##  6719
##  gap> DigitsNumber(6719, 16);
##  "1A3F"
##  </Example>
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
##  
InstallGlobalFunction(NumberDigits, function(str, base)
  local   res,  x,  nr, sign;
  res := 0;
  sign := 1;
  for x in str do
    nr := INT_CHAR(x) - 48;
    if nr = -3 then 
      # '-'
      sign := -sign;
    else
      if nr>48 then 
        nr := nr - 39;
      elif nr>15 then
        nr := nr - 7;
      fi;
      res := res*base + nr;
    fi;
  od;
  return sign*res;
end);

InstallGlobalFunction(DigitsNumber, function(n, base)
  local str, s;
  s := "";
  if n<0 then
    Add(s, '-');
    n := -n;
  fi;
  str := "";
  while n <> 0 do
    Add(str, HEXDIGITS[(n mod base) + 1]);
    n := QuoInt(n, base);
  od;
  return Concatenation(s, Reversed(str));
end);

 
##  <#GAPDoc Label="StripBeginEnd">
##  <ManSection >
##  <Func Arg="list, strip" Name="StripBeginEnd" />
##  <Returns>changed string</Returns>
##  <Description>
##  Here <A>list</A>  and <A>strip</A>  must be lists.  This function
##  returns the  sublist of list  which does not contain  the leading
##  and trailing  entries which are  entries of <A>strip</A>.  If the
##  result  is  equal  to  <A>list</A>  then  <A>list</A>  itself  is
##  returned.
##  
##  <Example>
##  gap> StripBeginEnd(" ,a, b,c,   ", ", ");
##  "a, b,c"
##  </Example>
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
##  
InstallGlobalFunction(StripBeginEnd, function(str, chars)
  local   pb,  l,  pe;
  pb := 1;
  l := Length(str);
  while  pb <= l and str[pb] in chars do
    pb := pb + 1;
  od;
  pe := l;
  while pe > 0 and str[pe] in chars do
    pe := pe - 1;
  od;
  if pb > 1 or pe < l then
    return str{[pb..pe]};
  else
    return str;
  fi;
end);


##  <#GAPDoc Label="NormalizedWhitespace">
##  <ManSection >
##  <Func Arg="str" Name="NormalizedWhitespace" />
##  <Returns>new string with white space normalized</Returns>
##  <Description>
##  This  function  gets  a  string  <A>str</A>  and  returns  a  new
##  string  which  is a  copy  of  <A>str</A> with  normalized  white
##  space.  Note  that  the   library  function  <Ref  BookName="ref"
##  Func="NormalizeWhitespace"  />  works in place  and  changes  its
##  argument.
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
##  
# moved into GAP library
##  InstallGlobalFunction(NormalizedWhitespace, function(str)
##    local   res;
##    res := ShallowCopy(str);
##    NormalizeWhitespace(res);
##    return res;
##  end);



##  <#GAPDoc Label="WrapTextAttribute">
##  <ManSection >
##  <Func Arg="str, attr" Name="WrapTextAttribute" />
##  <Returns>a string with markup</Returns>
##  <Description>
##  The argument <A>str</A> must be a text as &GAP; string, possibly with 
##  markup by escape sequences as in <Ref Var="TextAttr" />. This function
##  returns a string which is wrapped by the escape sequences <A>attr</A>
##  and <C>TextAttr.reset</C>. It takes care of markup in the given string
##  by appending <A>attr</A> also after each given <C>TextAttr.reset</C> in
##  <A>str</A>.
##  <Example>
##  gap> str := Concatenation("XXX",TextAttr.2, "BLUB", TextAttr.reset,"YYY");
##  "XXX\033[32mBLUB\033[0mYYY"
##  gap> str2 := WrapTextAttribute(str, TextAttr.1);
##  "\033[31mXXX\033[32mBLUB\033[0m\033[31mYYY\033[0m"
##  gap> str3 := WrapTextAttribute(str, TextAttr.underscore);
##  "\033[4mXXX\033[32mBLUB\033[0m\033[4mYYY\033[0m"
##  gap> # use Print(str); and so on to see how it looks like.
##  </Example>
##  </Description>
##  </ManSection>
##  
##  <#/GAPDoc>
InstallGlobalFunction(WrapTextAttribute, function(str, attr)
  str := SubstitutionSublist(str, TextAttr.reset, Concatenation(
            TextAttr.reset, attr));
  return Concatenation(attr, str, TextAttr.reset);
end);

##  <#GAPDoc Label="FormatParagraph">
##  <ManSection >
##  <Func Arg="str[, len][, flush][, attr][, widthfun]]]" 
##      Name="FormatParagraph" />
##  <Returns>the formatted paragraph as string</Returns>
##  <Description>
##  This function formats a text given  in the string <A>str</A> as a
##  paragraph. The optional arguments have the following meaning:
##  
##  <List >
##  <Mark><A>len</A></Mark>
##  <Item>the length of  the lines of the resulting  text (default is
##  <C>78</C>)</Item>
##  <Mark><A>flush</A></Mark>
##  <Item>can  be <C>"left"</C>,  <C>"right"</C>, <C>"center"</C>  or
##  <C>"both"</C>, telling that lines should be flushed left, flushed
##  right, centered or left-right justified, respectively (default is
##  <C>"both"</C>)</Item>
##  <Mark><A>attr</A></Mark>
##  <Item>is a  list of two strings;  the first is prepended  and the
##  second  appended to  each line  of  the result  (can for  example
##  be  used  for  indenting,  <C>["  ",  ""]</C>,  or  some  markup,
##  <C>[TextAttr.bold,   TextAttr.reset]</C>,   default  is   <C>["",
##  ""]</C>)</Item>
##  <Mark><A>widthfun</A></Mark>
##  <Item>must be a function which returns the display width of text in 
##  <A>str</A>. The default is <C>Length</C> assuming that each byte 
##  corresponds to a character of width one. If <A>str</A> is given in 
##  <C>UTF-8</C> encoding one can use <Ref Func="WidthUTF8String"/> here.
##  </Item>
##  </List>
##  
##  This function  tries to handle  markup with the  escape sequences
##  explained in <Ref Var="TextAttr"/> correctly.
##  
##  <Example>
##  gap> str := "One two three four five six seven eight nine ten eleven.";;
##  gap> Print(FormatParagraph(str, 25, "left", ["/* ", " */"]));           
##  /* One two three four five */
##  /* six seven eight nine ten */
##  /* eleven. */
##  </Example>
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
##  
BindGlobal("SPACESTRINGS", [" "]);
InstallGlobalFunction(FormatParagraph, function(arg)
  local   str,  len,  flush,  attr, width, i, words, esc, l, j, k, lw,  
          lines,  s,  ss,  nsp,  res,  a,  new,  qr,  b;
  str := arg[1];
  # default line length
  len := 78;
  # default flush (flush left and right)
  flush := "both";
  # default attribute (empty)
  attr := false;
  # default width function assumes that one byte is one character
  width := Length;
  # scan further arg's
  for i in [2..Length(arg)] do
    if IsInt(arg[i]) then
      len := arg[i];
    elif arg[i] in ["both", "left", "right", "center"] then
      flush := arg[i];
    elif IsList(arg[i]) then
      attr := arg[i];
    elif IsFunction(arg[i]) then
      width := arg[i];
    else
      Error("wrong argument", arg[i]);
    fi;
  od;
  for i in [Length(SPACESTRINGS)+1..len] do
    SPACESTRINGS[i] := Concatenation(SPACESTRINGS[i-1], " ");
  od;
  # we scan the string
  words := [];
  i := 1;
  esc := CHAR_INT(27);
  l := Length(str);
  while i<=l do
    if str[i] in WHITESPACE then
      # delete leading whitespace
      if Length(words)>0 then
        Add(words, 1);
      fi;
      i := i+1;
      while i<=l and str[i] in WHITESPACE do
        i := i+1;
      od;
    elif str[i] = esc then
      # sequences starting with ESC and stopping with the first letter
      # afterwards are not changed and considered to have length zero
      j := i+1;
      while j<=l and not str[j] in SMALLLETTERS and not str[j] in
        CAPITALLETTERS do
        j := j+1;
      od;
      if j>l then
        Error("string end inside escape sequence");
      else
        Add(words, [0, [i..j]]);
      fi;
      i := j+1;
    else
      j := i+1;
      while j<=l and not (str[j] in WHITESPACE or str[j]=esc) do
        j := j+1;
      od;
      if ForAll([i..j-1], k-> IsChar(str[k])) then
        Add(words, [width(str{[i..j-1]}), [i..j-1]]);
      else
        Add(words, [j-i, [i..j-1]]);
      fi;
      i := j;
    fi;
  od;
  # remove trailing white space
  lw := Length(words);
  if lw>0 and IsInt(words[lw]) then
    Unbind(words[lw]);
  fi;
  
  # split into lines
  lines := [];
  i := 1;
  lw := Length(words);
  while i <= lw do
    s := words[i][1];
    j := i+1;
    nsp := 0;
    while j <= lw and s+nsp < len do
      if IsInt(words[j]) then
        nsp := nsp+1;
        j := j+1;
      else
        # line breaks only at white space
        ss := s+nsp;
        k := j;
        while k <= lw and IsList(words[k]) do
          ss := ss+words[k][1];
          k := k+1;
        od;
        if s=0 or ss <= len then
          s := ss-nsp;
          j := k;
        else
          break;
        fi;
      fi;
    od;
    if IsInt(words[j-1]) then
      Add(lines, [s,nsp-1,[i..j-2]]);
      i := j;
    else
      Add(lines, [s,nsp,[i..j-1]]);
      i := j+1;
    fi;
  od;
  
  # format lines
  res := "";
  for i in [1..Length(lines)] do
    a := lines[i];
    new := words{a[3]};
    # now fill with spaces
    nsp := len - a[1] - a[2];
    if nsp > 0 then
      if flush = "right" then
        new := Concatenation([nsp], new);
      elif flush = "both" and a[2] > 0 and i < Length(lines) then
        qr := QuotientRemainder(nsp, a[2]);
        for j in [1..Length(new)] do
          if IsInt(new[j]) then
            if qr[2]>0 then
              new[j] := new[j]+qr[1]+1;
              qr[2] := qr[2]-1;
            else
              new[j] := new[j]+qr[1];
            fi;
          fi;
        od;
      elif flush = "center" and nsp > 1 then
        new := Concatenation([QuoInt(nsp,2)], new);
      fi;
    fi;
    # add text attribute begin
    if attr <> false and Length(new)>0 then
      if IsInt(new[1]) then
        new := Concatenation([new[1]], [attr[1]], new{[2..Length(new)]});
      else
        Append(res, attr[1]);
      fi;
    fi;
    s := "";
    for b in new do
      if IsInt(b) then
        Append(s, SPACESTRINGS[b]);
      elif IsString(b) then
        Append(s, b);
      else # range
        Append(s, str{b[2]});
      fi;
    od;
    # add text attribute begin after each text attribute reset (if it
    # is an escape sequence) 
    # and the end attribute
    if attr <> false then
      if Length(attr[1])>2 and attr[1]{[1,2]} = TextAttr.CSI then
        s := SubstitutionSublist(s, TextAttr.reset, attr[1]);
      fi;
      Append(s, attr[2]);
    fi;
    Add(s, '\n');
    Append(res, s);
  od;
  return res;
end);

##  <#GAPDoc Label="StripEscapeSequences">
##  <ManSection >
##  <Func Arg="str" Name="StripEscapeSequences" />
##  <Returns>string without escape sequences</Returns>
##  <Description>
##  This  function  returns  the  string one  gets  from  the  string
##  <A>str</A> by  removing all escape sequences  which are explained
##  in <Ref Var="TextAttr"/>.  If <A>str</A> does not  contain such a
##  sequence then <A>str</A> itself is returned.
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
##  
InstallGlobalFunction(StripEscapeSequences, function(str)
  local   esc,  res,  i,  ls,  p;
  esc := CHAR_INT(27);
  res := "";
  i := 1;
  ls := Length(str);
  while i <= ls do
    if str[i] = esc then
      i := i+1;
      while not str[i] in LETTERS do
        i := i+1;
      od;
      # first letter is last character of escape sequence
      i := i+1; 
    else
      p := Position(str, esc, i);
      if p=fail then
        if i=1 then
          # don't copy if no escape there
          return str;
        else
          Append(res, str{[i..ls]});
          return res;
        fi;
      else
        Append(res, str{[i..p-1]});
        i := p;
      fi;
    fi;
  od;
  return res;
end);
InstallGlobalFunction(SubstituteEscapeSequences, function(str, subs)
  local esc, res, i, ls, seq, pos, p;
  esc := CHAR_INT(27);
  res := "";
  i := 1;
  ls := Length(str);
  while i <= ls do
    if str[i] = esc and IsBound(str[i+1]) and str[i+1] = '[' then
      seq := "";
      i := i+1;
      while not str[i] in LETTERS do
        i := i+1;
        Add(seq, str[i]);
      od;
      # first letter is last character of escape sequence
      i := i+1; 
      pos := PositionSet(subs[1], seq);
      if pos <> fail then
        seq := subs[2][pos];
      else
        Add(res, esc);
        Add(res, '[');
      fi;
      Append(res, seq);
    else
      p := Position(str, esc, i);
      if p=fail then
        if i=1 then
          # don't copy if no escape there
          return str;
        else
          Append(res, str{[i..ls]});
          return res;
        fi;
      else
        Append(res, str{[i..p-1]});
        i := p;
      fi;
    fi;
  od;
  return res;
end);



##  <#GAPDoc Label="WordsString">
##  <ManSection >
##  <Func Arg="str" Name="WordsString" />
##  <Returns>list of strings containing the words</Returns>
##  <Description>
##  This returns  the list of  words of a  text stored in  the string
##  <A>str</A>. All non-letters are considered as word boundaries and
##  are removed.
##  <Example>
##  gap> WordsString("one_two \n    three!?");
##  [ "one", "two", "three" ]
##  </Example>
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
##  
InstallGlobalFunction(WordsString, function(str)
  local   nonletters, wds;
  nonletters := Set("0123456789 \n\r\t\b+*~^\\\"#'`'/?-_.:,;<>|=()[]{}&%$§!");
  wds := SplitString(str, "", nonletters);
  return wds;
end);

# The GAP library will contain a new function CrcString. To make GAPDoc
# running with current/older versions of GAP we use the following helper,
# if necessary with a simple fallback.
if IsBoundGlobal("CrcString") then
  InstallGlobalFunction(CrcText, CrcString); 
else
  InstallGlobalFunction(CrcText, function(s)
    local n, res;
    n := "guckCRCXQWYNVOH";
    FileString(n, s);
    res := CrcFile(n);
    RemoveFile(n);
    return res;
  end);
fi;

##  <#GAPDoc Label="Base64String">
##  <ManSection >
##  <Func Arg="str" Name="Base64String" />
##  <Func Arg="bstr" Name="StringBase64" />
##  <Returns>a string</Returns>
##  <Description>
##  The  first  function  translates  arbitrary   binary  data  given  as  a
##  GAP  string  into   a  <E>base  64</E>  encoded   string.  This  encoded
##  string  contains  only  printable  ASCII   characters  and  is  used  in
##  various  data  transfer  protocols  (<C>MIME</C>  encoded  emails,  weak
##  password   encryption,  ...).   We   use  the   specification  in   <URL
##  Text="RFC&#160;2045">http://tools.ietf.org/html/rfc2045</URL>.<P/>
##  
##  The second function  has the reverse functionality. Here  we also accept
##  the characters  <C>-_</C> instead of  <C>+/</C> as last  two 
##  characters.  Whitespace is ignored.
##  
##  <Example>
##  gap> b := Base64String("This is a secret!");
##  "VGhpcyBpcyBhIHNlY3JldCEA="
##  gap> StringBase64(b);                       
##  "This is a secret!"
##  </Example>
##  </Description>
##  </ManSection>
##  <#/GAPDoc>
BindGlobal("Base64LETTERS",
          "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/");
BindGlobal("Base64REVERSE",
[,,,,,,,,,-1,,,-1,,,,,,,,,,,,,,,,,,,-1,,,,,,,,,,,62,,62,,63,52,53,54,55,56,57,58,59,60,61,,,,-2,,,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,,,,,63,,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51]);
InstallGlobalFunction(Base64String, function(str)
  local istr, pad, i, res, a, d, c, b;
  istr := INTLIST_STRING(str, 1);
  pad := (-Length(istr)) mod 3;
  for i in [1..pad] do
    Add(istr,0);
  od;
  i := 1;
  res := [];
  while i < Length(istr) do
    if i > 1 and i mod 57 = 1 then
      Add(res, '\n');
    fi;
    a := (istr[i]*256+istr[i+1])*256+istr[i+2];
    d := RemInt(a, 64);
    a := (a-d)/64;
    c := RemInt(a, 64);
    a := (a-c)/64;
    b := RemInt(a, 64);
    a := (a-b)/64;
    Append(res, Base64LETTERS{[a,b,c,d]+1});
    i := i+3;
  od;
  if i mod 57 = 1 and pad > 0 then
    Add(res, '\n');
  fi;
  for i in [1..pad] do
    Add(res, '=');
  od;
  return res;
end);
 
InstallGlobalFunction(StringBase64, function(bstr)
  local istr, res, j, n, d, c, a;
  istr := Base64REVERSE{INTLIST_STRING(bstr, 1)};
  res := [];
  j := 0;
  n := 0;
  for a in istr do
    if a <> -1 then
      if a = -2 then
        Unbind(res[Length(res)]);
      else
        n := n*64+a;
        j := j+1;
        if j = 4 then
          d := RemInt(n, 256);
          n := (n-d)/256;
          c := RemInt(n, 256);
          n := (n-c)/256;
          Append(res, [n,c,d]);
          j := 0;
          n := 0;
        fi;
      fi;
    fi;
  od;
  return STRING_SINTLIST(res);
end);


