Interned strings unification for TS/NTS

Hereby, interned strings are supported in thread safe PHP. The patch
implements two types of interned strings

- interning per process, strings are not freed till process end
- interning per request, strings are freed at request end

There is no runtime interning.

With Opcache, all the permanent iterned strings are copied into SHM on
startup, additional copying into SHM might happen on demand.

1 files changed, 157 insertions, 145 deletions

diff --git a/Zend/zend_string.c b/Zend/zend_string.c
index dbc44870c2..d3fbdf2f19 100644
--- a/Zend/zend_string.c
+++ b/Zend/zend_string.c
@@ -22,48 +22,33 @@
 #include "zend_globals.h"
 
 ZEND_API zend_string *(*zend_new_interned_string)(zend_string *str);
-ZEND_API void (*zend_interned_strings_snapshot)(void);
-ZEND_API void (*zend_interned_strings_restore)(void);
 
-static zend_string *zend_new_interned_string_int(zend_string *str);
-static void zend_interned_strings_snapshot_int(void);
-static void zend_interned_strings_restore_int(void);
+static zend_string *zend_new_interned_string_permanent(zend_string *str);
+static zend_string *zend_new_interned_string_request(zend_string *str);
+
+/* Any strings interned in the startup phase. Common to all the threads,
+   won't be free'd until process exit. If we want an ability to 
+   add permanent strings even after startup, it would be still
+   possible on costs of locking in the thread safe builds. */
+static HashTable interned_strings_permanent;
+
+static zend_new_interned_string_func_t interned_string_request_handler = zend_new_interned_string_request;
+static zend_string_copy_storage_func_t interned_string_copy_storage = NULL;
+
+ZEND_API zend_string  *zend_empty_string = NULL;
+ZEND_API zend_string  *zend_one_char_string[256];
+ZEND_API zend_string **zend_known_strings = NULL;
 
 ZEND_API zend_ulong zend_hash_func(const char *str, size_t len)
 {
 	return zend_inline_hash_func(str, len);
 }
 
-#ifndef ZTS
 static void _str_dtor(zval *zv)
 {
 	zend_string *str = Z_STR_P(zv);
 	pefree(str, GC_FLAGS(str) & IS_STR_PERSISTENT);
 }
-#endif
-
-/* Readonly, so assigned also per thread. */
-static const zend_string **known_interned_strings = NULL;
-static uint32_t known_interned_strings_count = 0;
-
-ZEND_API uint32_t zend_intern_known_strings(const char **strings, uint32_t count)
-{
-	uint32_t i, old_count = known_interned_strings_count;
-
-	known_interned_strings = perealloc(known_interned_strings, sizeof(char*) * (old_count + count), 1);
-	for (i = 0; i < count; i++) {
-#ifndef ZTS
-		zend_string *str = zend_string_init(strings[i], strlen(strings[i]), 1);
-		known_interned_strings[known_interned_strings_count + i] =
-			zend_new_interned_string_int(str);
-#else
-		known_interned_strings[known_interned_strings_count + i] =
-			zend_zts_interned_string_init(strings[i], strlen(strings[i]));
-#endif
-	}
-	known_interned_strings_count = old_count + count;
-	return old_count;
-}
 
 static const char *known_strings[] = {
 #define _ZEND_STR_DSC(id, str) str,
@@ -72,175 +57,202 @@ ZEND_KNOWN_STRINGS(_ZEND_STR_DSC)
 	NULL
 };
 
-void zend_known_interned_strings_init(zend_string ***strings, uint32_t *count)
+static void zend_init_interned_strings_ht(HashTable *interned_strings, int permanent)
 {
-	*strings = (zend_string **)known_interned_strings;
-	*count   = known_interned_strings_count;
+	zend_hash_init(interned_strings, 1024, NULL, _str_dtor, permanent);
+
+	interned_strings->nTableMask = -interned_strings->nTableSize;
+	HT_SET_DATA_ADDR(interned_strings, pemalloc(HT_SIZE(interned_strings), permanent));
+	HT_HASH_RESET(interned_strings);
+	interned_strings->u.flags |= HASH_FLAG_INITIALIZED;
 }
 
-void zend_interned_strings_init(void)
+ZEND_API void zend_interned_strings_init(void)
 {
-#ifndef ZTS
+	char s[2];
+	int i;
 	zend_string *str;
 
-	zend_hash_init(&CG(interned_strings), 1024, NULL, _str_dtor, 1);
+	zend_init_interned_strings_ht(&interned_strings_permanent, 1);
 
-	CG(interned_strings).nTableMask = -CG(interned_strings).nTableSize;
-	HT_SET_DATA_ADDR(&CG(interned_strings), pemalloc(HT_SIZE(&CG(interned_strings)), 1));
-	HT_HASH_RESET(&CG(interned_strings));
-	CG(interned_strings).u.flags |= HASH_FLAG_INITIALIZED;
+	zend_new_interned_string = zend_new_interned_string_permanent;
 
 	/* interned empty string */
 	str = zend_string_alloc(sizeof("")-1, 1);
 	ZSTR_VAL(str)[0] = '\000';
-	CG(empty_string) = zend_new_interned_string_int(str);
-#endif
+	zend_empty_string = zend_new_interned_string_permanent(str);
 
-	/* one char strings (the actual interned strings are going to be created by ext/opcache) */
-	memset(CG(one_char_string), 0, sizeof(CG(one_char_string)));
+	s[1] = 0;
+	for (i = 0; i < 256; i++) {
+		s[0] = i;
+		zend_one_char_string[i] = zend_new_interned_string_permanent(zend_string_init(s, 1, 1));
+	}
 
 	/* known strings */
-	zend_intern_known_strings(known_strings, (sizeof(known_strings) / sizeof(known_strings[0])) - 1);
-	zend_known_interned_strings_init(&CG(known_strings), &CG(known_strings_count));
-
-	zend_new_interned_string = zend_new_interned_string_int;
-	zend_interned_strings_snapshot = zend_interned_strings_snapshot_int;
-	zend_interned_strings_restore = zend_interned_strings_restore_int;
+	zend_known_strings = pemalloc(sizeof(zend_string*) * ((sizeof(known_strings) / sizeof(known_strings[0]) - 1)), 1);
+	for (i = 0; i < (sizeof(known_strings) / sizeof(known_strings[0])) - 1; i++) {
+		str = zend_string_init(known_strings[i], strlen(known_strings[i]), 1);
+		zend_known_strings[i] = zend_new_interned_string_permanent(str);
+	}
 }
 
-void zend_interned_strings_dtor(void)
+ZEND_API void zend_interned_strings_dtor(void)
 {
-#ifndef ZTS
-	zend_hash_destroy(&CG(interned_strings));
-#else
-	uint32_t i;
+	zend_hash_destroy(&interned_strings_permanent);
 
-	for (i = 0; i < CG(known_strings_count); i++) {
-		zend_zts_interned_string_free(&CG(known_strings)[i]);
-	}
-#endif
-	free(CG(known_strings));
-	CG(known_strings) = NULL;
-	CG(known_strings_count) = 0;
-	known_interned_strings = NULL;
-	known_interned_strings_count = 0;
+	free(zend_known_strings);
+	zend_known_strings = NULL;
 }
 
-static zend_string *zend_new_interned_string_int(zend_string *str)
+static zend_always_inline zend_string *zend_interned_string_ht_lookup(zend_string *str, HashTable *interned_strings)
 {
-#ifndef ZTS
 	zend_ulong h;
 	uint32_t nIndex;
 	uint32_t idx;
 	Bucket *p;
 
-	if (ZSTR_IS_INTERNED(str)) {
-		return str;
-	}
-
 	h = zend_string_hash_val(str);
-	nIndex = h | CG(interned_strings).nTableMask;
-	idx = HT_HASH(&CG(interned_strings), nIndex);
+	nIndex = h | interned_strings->nTableMask;
+	idx = HT_HASH(interned_strings, nIndex);
 	while (idx != HT_INVALID_IDX) {
-		p = HT_HASH_TO_BUCKET(&CG(interned_strings), idx);
+		p = HT_HASH_TO_BUCKET(interned_strings, idx);
 		if ((p->h == h) && (ZSTR_LEN(p->key) == ZSTR_LEN(str))) {
 			if (!memcmp(ZSTR_VAL(p->key), ZSTR_VAL(str), ZSTR_LEN(str))) {
-				zend_string_release(str);
 				return p->key;
 			}
 		}
 		idx = Z_NEXT(p->val);
 	}
 
+	return NULL;
+}
+
+static zend_never_inline void zend_string_table_grow(HashTable *interned_strings)
+{
+	if (EXPECTED(interned_strings->nTableSize < HT_MAX_SIZE)) {	/* Let's double the table size */
+		void *new_data;
+		void *old_data = HT_GET_DATA_ADDR(interned_strings);
+		Bucket *old_buckets = interned_strings->arData;
+
+		interned_strings->nTableSize += interned_strings->nTableSize;
+		interned_strings->nTableMask = -interned_strings->nTableSize;
+		new_data = pemalloc(HT_SIZE(interned_strings), interned_strings->u.flags & HASH_FLAG_PERSISTENT);
+
+		HT_SET_DATA_ADDR(interned_strings, new_data);
+		memcpy(interned_strings->arData, old_buckets, sizeof(Bucket) * interned_strings->nNumUsed);
+		pefree(old_data, interned_strings->u.flags & HASH_FLAG_PERSISTENT);
+		zend_hash_rehash(interned_strings);
+	}
+}
+
+/* This function might be not thread safe at least because it would update the
+   hash val in the passed string. Be sure it is called in the appropriate context. */
+static zend_always_inline zend_string *zend_add_interned_string(zend_string *str, HashTable *interned_strings, uint32_t flags)
+{
+	zend_ulong h;
+	uint32_t nIndex;
+	uint32_t idx;
+	Bucket *p;
+
+	h = zend_string_hash_val(str);
+
 	GC_REFCOUNT(str) = 1;
-	GC_FLAGS(str) |= IS_STR_INTERNED;
-
-	if (CG(interned_strings).nNumUsed >= CG(interned_strings).nTableSize) {
-		if (CG(interned_strings).nTableSize < HT_MAX_SIZE) {	/* Let's double the table size */
-			void *new_data;
-			void *old_data = HT_GET_DATA_ADDR(&CG(interned_strings));
-			Bucket *old_buckets = CG(interned_strings).arData;
-
-			CG(interned_strings).nTableSize += CG(interned_strings).nTableSize;
-			CG(interned_strings).nTableMask = -CG(interned_strings).nTableSize;
-			new_data = malloc(HT_SIZE(&CG(interned_strings)));
-
-			if (new_data) {
-				HT_SET_DATA_ADDR(&CG(interned_strings), new_data);
-				memcpy(CG(interned_strings).arData, old_buckets, sizeof(Bucket) * CG(interned_strings).nNumUsed);
-				free(old_data);
-				zend_hash_rehash(&CG(interned_strings));
-			} else {
-				CG(interned_strings).nTableSize = CG(interned_strings).nTableSize >> 1;
-				CG(interned_strings).nTableMask = -CG(interned_strings).nTableSize;
-			}
-		}
+	GC_FLAGS(str) |= IS_STR_INTERNED | flags;
+
+	if (UNEXPECTED(interned_strings->nNumUsed >= interned_strings->nTableSize)) {
+		zend_string_table_grow(interned_strings);
 	}
 
-	idx = CG(interned_strings).nNumUsed++;
-	CG(interned_strings).nNumOfElements++;
-	p = CG(interned_strings).arData + idx;
+	idx = interned_strings->nNumUsed++;
+	interned_strings->nNumOfElements++;
+	p = interned_strings->arData + idx;
 	p->h = h;
 	p->key = str;
 	Z_STR(p->val) = str;
 	Z_TYPE_INFO(p->val) = IS_INTERNED_STRING_EX;
-	nIndex = h | CG(interned_strings).nTableMask;
-	Z_NEXT(p->val) = HT_HASH(&CG(interned_strings), nIndex);
-	HT_HASH(&CG(interned_strings), nIndex) = HT_IDX_TO_HASH(idx);
+	nIndex = h | interned_strings->nTableMask;
+	Z_NEXT(p->val) = HT_HASH(interned_strings, nIndex);
+	HT_HASH(interned_strings, nIndex) = HT_IDX_TO_HASH(idx);
 
 	return str;
-#else
-	return str;
-#endif
 }
 
-static void zend_interned_strings_snapshot_int(void)
+ZEND_API zend_string *zend_interned_string_find_permanent(zend_string *str)
 {
-#ifndef ZTS
-	uint32_t idx;
-	Bucket *p;
+	return zend_interned_string_ht_lookup(str, &interned_strings_permanent);
+}
+
+
+static zend_string *zend_new_interned_string_permanent(zend_string *str)
+{
+	zend_string *ret;
+
+	if (ZSTR_IS_INTERNED(str)) {
+		return str;
+	}
 
-	idx = CG(interned_strings).nNumUsed;
-	while (idx > 0) {
-		idx--;
-		p = CG(interned_strings).arData + idx;
-		ZEND_ASSERT(GC_FLAGS(p->key) & IS_STR_PERSISTENT);
-		GC_FLAGS(p->key) |= IS_STR_PERMANENT;
+	ret = zend_interned_string_ht_lookup(str, &interned_strings_permanent);
+	if (ret) {
+		zend_string_release(str);
+		return ret;
 	}
-#endif
+
+	return zend_add_interned_string(str, &interned_strings_permanent, IS_STR_PERMANENT);
 }
 
-static void zend_interned_strings_restore_int(void)
+static zend_string *zend_new_interned_string_request(zend_string *str)
 {
-#ifndef ZTS
-	uint32_t nIndex;
-	uint32_t idx;
-	Bucket *p;
+	zend_string *ret;
+
+	if (ZSTR_IS_INTERNED(str)) {
+		return str;
+	}
+
+	/* Check for permanent strings, the table is readonly at this point. */
+	ret = zend_interned_string_ht_lookup(str, &interned_strings_permanent);
+	if (ret) {
+		zend_string_release(str);
+		return ret;
+	}
+
+	ret = zend_interned_string_ht_lookup(str, &CG(interned_strings));
+	if (ret) {
+		zend_string_release(str);
+		return ret;
+	}
+
+	/* Create a short living interned, freed after the request. */
+	ret = zend_add_interned_string(str, &CG(interned_strings), 0);
+
+	return ret;
+}
+
+ZEND_API void zend_interned_strings_activate(void)
+{
+	zend_init_interned_strings_ht(&CG(interned_strings), 0);
+}
 
-	idx = CG(interned_strings).nNumUsed;
-	while (idx > 0) {
-		idx--;
-		p = CG(interned_strings).arData + idx;
-		if (GC_FLAGS(p->key) & IS_STR_PERMANENT) break;
-		CG(interned_strings).nNumUsed--;
-		CG(interned_strings).nNumOfElements--;
-
-		GC_FLAGS(p->key) &= ~IS_STR_INTERNED;
-		GC_REFCOUNT(p->key) = 1;
-		zend_string_free(p->key);
-
-		nIndex = p->h | CG(interned_strings).nTableMask;
-		if (HT_HASH(&CG(interned_strings), nIndex) == HT_IDX_TO_HASH(idx)) {
-			HT_HASH(&CG(interned_strings), nIndex) = Z_NEXT(p->val);
-		} else {
-			uint32_t prev = HT_HASH(&CG(interned_strings), nIndex);
-			while (Z_NEXT(HT_HASH_TO_BUCKET(&CG(interned_strings), prev)->val) != idx) {
-				prev = Z_NEXT(HT_HASH_TO_BUCKET(&CG(interned_strings), prev)->val);
- 			}
-			Z_NEXT(HT_HASH_TO_BUCKET(&CG(interned_strings), prev)->val) = Z_NEXT(p->val);
- 		}
- 	}
-#endif
+ZEND_API void zend_interned_strings_deactivate(void)
+{
+	zend_hash_destroy(&CG(interned_strings));
+}
+
+ZEND_API void zend_interned_strings_set_request_storage_handler(zend_new_interned_string_func_t handler)
+{
+	interned_string_request_handler = handler;
+}
+
+ZEND_API void zend_interned_strings_set_permanent_storage_copy_handler(zend_string_copy_storage_func_t handler)
+{
+	interned_string_copy_storage = handler;
+}
+
+ZEND_API void zend_interned_strings_switch_storage(void)
+{
+	if (interned_string_copy_storage) {
+		interned_string_copy_storage();
+	}
+	zend_new_interned_string = interned_string_request_handler;
 }
 
 /*