Changeset 26 for trunk/Classes/serialization/tokenizer.hpp

Timestamp:

2010-05-28 07:39:52 (2 years ago)

Author:

kpoole

Message:

Episode 5

File:

• trunk/Classes/serialization/tokenizer.hpp (modified) (10 diffs)

trunk/Classes/serialization/tokenizer.hpp

@@ -23 +23 @@
 #include <string>
 
+#define TOKENIZER_BUFFER_SIZE   50 * 1024 * 1024
+
 class config;
 
@@ -31 +33 @@
         leading_spaces(),
         value()
-        {}
+        {
+            reset();
+        }
 
     enum token_type {
@@ -53 +57 @@
         value.clear();
         leading_spaces.clear();
+        /*
+        value_ptr = 0;
+        value_size = 0;
+        leading_spaces_ptr = 0;
+        leading_spaces_size = 0;
+         */
     }
 
     std::string leading_spaces;
     std::string value;
+    
+    /*
+    // KP: creating std::strings all the time was a huge performance hit
+    const char *leading_spaces_ptr;
+    int leading_spaces_size;
+    const char *value_ptr;
+    int value_size;
+    */
 };
 
@@ -64 +82 @@
     public:
         tokenizer(std::istream& in);
-        ~tokenizer() {}
+        ~tokenizer();
 
         const token& next_token();  // KP: moved to tokenizer.cpp
@@ -71 +89 @@
         {
             if(token_.value == "_")
+            //if(token_.value_ptr && token_.value_ptr[0] == '_')
                 token_.type = token::token_type('_');
         }
@@ -115 +134 @@
             do {
                 if (LIKELY(in_.good()))
+                //if (curPos < numChars-1)
                 {
                     current_ = in_.get();
+                    //current_ = (unsigned char) in_buffer[curPos++];
                 }
                 else
@@ -123 +144 @@
                     return;
                 }
-            }while (UNLIKELY(current_ == '\r'));
+            } while (UNLIKELY(current_ == '\r'));
+                        
 #if 0
             // @todo: disabled untill campaign server is fixed
@@ -146 +168 @@
         {
             return in_.peek();
+            //return in_buffer[curPos];
         }
 
@@ -153 +176 @@
             return c == ' ' || c == '\t';
         }
-        inline bool is_alnum(const int c) const
-        {
-            return (c >= 'a' && c <= 'z')
-                || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
-        }
+    /*inline*/ bool is_alnum(const int c) const;
+        
         void skip_comment();
 
@@ -167 +187 @@
 #endif
         std::istream& in_;
+    
+        /*
+        // KP: added for much faster tokenizing
+        char *in_buffer;
+        int curPos;
+        int numChars;
+         */
 };