20 years ago by Rolland Santimano — view source — reply

unread

Submitting patch for Unicode impl of strtok(), since I don't have a
proper test setup, as yet.

-- Rolland

20 years ago by Rolland Santimano — view source — reply

unread

--- Rolland Santimano rollandsantimano@yahoo.com wrote:

Submitting patch for Unicode impl of strtok(), since I don't have a
proper test setup, as yet.

-- Rolland

OK, attachment got stripped - please check inline patch.

Index: ext/standard/basic_functions.h

RCS file: /repository/php-src/ext/standard/basic_functions.h,v
retrieving revision 1.140
diff -u -r1.140 basic_functions.h
--- ext/standard/basic_functions.h 13 Aug 2005 02:23:29 -0000 1.140
+++ ext/standard/basic_functions.h 9 Sep 2005 18:13:25 -0000
@@ -153,9 +153,9 @@
HashTable *user_shutdown_function_names;
HashTable putenv_ht;
zval *strtok_zval;

```
  char *strtok_string;
```

  void *strtok_string;
  char *locale_string;

```
  char *strtok_last;
```

  void *strtok_last;
  char strtok_table[256];
  ulong strtok_len;
  char str_ebuf[40];

Index: ext/standard/string.c

RCS file: /repository/php-src/ext/standard/string.c,v
retrieving revision 1.481
diff -u -r1.481 string.c
--- ext/standard/string.c 8 Sep 2005 14:07:40 -0000 1.481
+++ ext/standard/string.c 9 Sep 2005 18:13:54 -0000
@@ -1315,88 +1315,168 @@
Tokenize a string */
PHP_FUNCTION(strtok)
{

```
  zval **args[2];
```
```
  zval **tok, **str;
```
```
  char *token;
```
```
  char *token_end;
```
```
  char *p;
```
```
  char *pe;
```

```
  void *tok, *str;
```
```
  int32_t tok_len, str_len;
```
```
  zend_uchar tok_type, str_type;
```
```
  zval *zv;
```
```
  char *token, *token_end, *p, *pe;
```
```
  UChar *u_token, *u_p, *u_pe;
```
```
  UChar32 ch, th;
```
```
  int32_t start, end, i, j, rem_len;
```

  int delim_found, token_present;
  int skipped = 0;

  if (ZEND_NUM_ARGS() < 1 || ZEND_NUM_ARGS() > 2 ||

zend_get_parameters_array_ex(ZEND_NUM_ARGS(), args) == FAILURE) {

  if (ZEND_NUM_ARGS() < 1 || ZEND_NUM_ARGS() > 2) {
          WRONG_PARAM_COUNT;
  }

  if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "T|T",

                                                    &str, &str_len, &str_type,

                                                    &tok, &tok_len, &tok_type) == FAILURE) {

```
          return;
```
```
  }
```

  switch (ZEND_NUM_ARGS()) {
          case 1:

```
                  tok = args[0];
```

```
                  tok = str;
```
```
                  tok_len = str_len;
```

                  tok_type = str_type;
                  break;

          default:
          case 2:

```
                  str = args[0];
```
```
                  tok = args[1];
```

                  convert_to_string_ex(str);

                  zval_add_ref(str);
                  if (BG(strtok_zval)) {
                          zval_ptr_dtor(&BG(strtok_zval));
                  }

                  BG(strtok_zval) = *str;

                  BG(strtok_last) = BG(strtok_string) = Z_STRVAL_PP(str);

                  BG(strtok_len) = Z_STRLEN_PP(str);

```
                  MAKE_STD_ZVAL(zv);
```

                  if (str_type == IS_UNICODE) {

                          ZVAL_UNICODEL(zv, (UChar *)str, str_len, 1);

                  } else if (str_type == IS_BINARY) {

                          ZVAL_BINARYL(zv, (char *)str, str_len, 1);

```
                  } else {
```

                          ZVAL_STRINGL(zv, (char *)str, str_len, 1);

```
                  }
```

                  BG(strtok_zval) = zv;

                  if (str_type == IS_UNICODE) {

                          BG(strtok_last) = BG(strtok_string) = Z_USTRVAL_P(zv);

```
                  } else {
```

                          BG(strtok_last) = BG(strtok_string) = Z_STRVAL_P(zv);

```
                  }
```

                  BG(strtok_len) = str_len;
                  break;
  }

  p = BG(strtok_last); /* Where we start to search */

  pe = BG(strtok_string) + BG(strtok_len);

```
  if (!p || p >= pe) {
```

  if (BG(strtok_zval) && tok_type != Z_TYPE_P(BG(strtok_zval))) {

          php_error_docref(NULL TSRMLS_CC, E_WARNING, "Delimiter type must match string

type.");
RETURN_FALSE;
}

```
  convert_to_string_ex(tok);
```
```
  token = Z_STRVAL_PP(tok);
```

  token_end = token + Z_STRLEN_PP(tok);

```
  if (tok_type == IS_UNICODE) {
```

          u_p = (UChar *)BG(strtok_last); /* Where we start to search */

          u_pe = (UChar *)BG(strtok_string) + BG(strtok_len);

```
          u_token = (UChar *)tok;
```
```
          if (!u_p || u_p >= u_pe) {
```
```
                  RETURN_FALSE;
```
```
          }
```
```
          rem_len = u_pe - u_p;
```

```
  while (token < token_end) {
```
```
          STRTOK_TABLE(token++) = 1;
```
```
  }
```
```
  /* Skip leading delimiters */
```
```
  while (STRTOK_TABLE(p)) {
```
```
          if (++p >= pe) {
```

                  /* no other chars left */

          /* Skip leading delimiters */

```
          token_present = 0;
```

          for (i = 0 ; (u_p + i) < u_pe ; ) {

```
                  delim_found = 0;
```

                  U16_NEXT(u_p, i, rem_len, ch);

                  for (j = 0 ; j < tok_len ; ) {

                          U16_NEXT(u_token, j, tok_len, th);

                          if ( ch == th ) {

                                  delim_found = 1;

                                  break;

```
                          }
```
```
                  }
```

                  if (delim_found == 0) {

                          U16_BACK_1(u_p, 0, i); /* U16_NEXT() post-incrs 'i' */

```
                          start = i;
```

                          token_present = 1;

```
                          break;
```
```
                  }
```
```
          }
```

          if (token_present == 0) {
                  BG(strtok_last) = NULL;

```
                  RETVAL_FALSE;
```
```
                  goto restore;
```

                  RETURN_FALSE;
          }

```
          skipped++;
```
```
  }
```

  /* We know at this place that *p is no delimiter, so skip it */

```
  while (++p < pe) {
```
```
          if (STRTOK_TABLE(p)) {
```

                  goto return_token;

```
          /* Seek to next delimiter */
```
```
          delim_found = 0;
```

          for (i = start ; (u_p + i) < u_pe ; ) {

                  U16_NEXT(u_p, i, rem_len, ch);

                  for (j = 0 ; j < tok_len ; ) {

                          U16_NEXT(u_token, j, tok_len, th);

                          if ( ch == th ) {

                                  delim_found = 1;

                                  break;

```
                          }
```
```
                  }
```
```
                  if (delim_found) {
```

                          U16_BACK_1(u_p, 0, i); /* 'i' was beyond delimiter */

```
                          break;
```
```
                  }
```
```
          }
```
```
          end = i;
```
```
          if (end - start) {
```

                  BG(strtok_last) = u_p + end;

                  RETURN_UNICODEL(u_p + start, end - start, 1);

```
          } else {
```

                  BG(strtok_last) = NULL;

                  RETURN_FALSE;
          }

```
  }
```
```
  if (p - BG(strtok_last)) {
```

-return_token:

          RETVAL_STRINGL(BG(strtok_last) + skipped, (p - BG(strtok_last)) - skipped, 1);

          BG(strtok_last) = p + 1;
  } else {

```
          RETVAL_FALSE;
```
```
          BG(strtok_last) = NULL;
```
```
  }
```

          p = (char *)BG(strtok_last); /* Where we start to search */

          pe = (char *)BG(strtok_string) + BG(strtok_len);

```
          if (!p || p >= pe) {
```
```
                  RETURN_FALSE;
```
```
          }
```
```
          token = (char *)tok;
```
```
          token_end = token + tok_len;
```
```
          while (token < token_end) {
```

                  STRTOK_TABLE(token++) = 1;

```
          }
```

  /* Restore table -- usually faster then memset'ing the table on every invocation */

          /* Skip leading delimiters */

```
          while (STRTOK_TABLE(p)) {
```
```
                  if (++p >= pe) {
```

                          /* no other chars left */

                          BG(strtok_last) = NULL;

                          RETVAL_FALSE;

                          goto restore;

```
                  }
```
```
                  skipped++;
```
```
          }
```

          /* We know at this place that *p is no delimiter, so skip it */

```
          while (++p < pe) {
```

                  if (STRTOK_TABLE(p)) {

                          goto return_token;

```
                  }
```
```
          }
```

          if (p - (char *)BG(strtok_last)) {

+return_token:

                  if (tok_type == IS_BINARY) {

                          RETVAL_BINARYL((char *)BG(strtok_last) + skipped, (p - (char

*)BG(strtok_last)) - skipped, 1);

```
                  } else {
```

                          RETVAL_STRINGL((char *)BG(strtok_last) + skipped, (p - (char

*)BG(strtok_last)) - skipped, 1);

```
                  }
```

                  BG(strtok_last) = p + 1;

```
          } else {
```
```
                  RETVAL_FALSE;
```

                  BG(strtok_last) = NULL;

```
          }
```

          /* Restore table -- usually faster then memset'ing the table on every invocation

*/
restore:

```
  token = Z_STRVAL_PP(tok);
```
```
  while (token < token_end) {
```
```
          STRTOK_TABLE(token++) = 0;
```

```
          token = (char *)tok;
```
```
          while (token < token_end) {
```

                  STRTOK_TABLE(token++) = 0;

```
          }
  }
```

}
/* }}} */

[PATCH] strtok()

OK, attachment got stripped - please check inline patch.

Index: ext/standard/basic_functions.h

Index: ext/standard/string.c