encoding stuff

Nathan Marz · Nathan Marz · commit 0aa8fc5e671e · 2012-03-21T05:47:58.000-07:00
diff --git a/src/dev/resources/tester_bolt.py b/src/dev/resources/tester_bolt.py
@@ -1,3 +1,5 @@
+# This Python file uses the following encoding: utf-8
+
 import storm
 from random import random
 
@@ -7,6 +9,7 @@ def initialize(self, conf, context):
 
     def process(self, tup):
         word = tup.values[0];
+        storm.log("BBB: " + word + " " + str(ord(word[-1])))
         if (random() < 0.75):
             storm.emit([word + 'lalala'], anchors=[tup])
             storm.ack(tup)
diff --git a/src/dev/resources/tester_spout.py b/src/dev/resources/tester_spout.py
@@ -1,9 +1,11 @@
+# This Python file uses the following encoding: utf-8
+
 from storm import Spout, emit, log
 from random import choice
 from time import sleep
 from uuid import uuid4
 
-words = ["nathan", "mike", "jackson", "golda", "bertels"]
+words = ["nathan", "mike", "jackson", "golda", "bertels人"]
 
 class TesterSpout(Spout):
     def initialize(self, conf, context):
@@ -15,6 +17,7 @@ def nextTuple(self):
         word = choice(words)
         id = str(uuid4())
         self.pending[id] = word
+        log("SSS: " + word + " " + str(ord(word[-1])))        
         emit([word], id=id)
 
     def ack(self, id):
diff --git a/src/jvm/backtype/storm/utils/ShellProcess.java b/src/jvm/backtype/storm/utils/ShellProcess.java
@@ -44,12 +44,11 @@ public void writeObject(Object obj) throws IOException {
         writeString(JSONValue.toJSONString(obj));
     }
 
-    public void writeString(String string) throws IOException {
-        // don't need synchronization for now
-        //synchronized (processIn) {
-            processIn.writeBytes(string + "\nend\n");
-            processIn.flush();
-            //}
+    public void writeString(String str) throws IOException {
+        byte[] strBytes = str.getBytes("UTF-8");
+        processIn.write(strBytes, 0, strBytes.length);
+        processIn.writeBytes("\nend\n");
+        processIn.flush();
     }
 
     // returns null for sync. odd?
diff --git a/src/multilang/py/storm.py b/src/multilang/py/storm.py
@@ -4,13 +4,12 @@
 from collections import deque
 
 try:
-    import cjson
-    json_encode = cjson.encode
-    json_decode = lambda x: cjson.decode(x, all_unicode=True)
+    import simplejson as json
 except ImportError:
     import json
-    json_encode = lambda x: json.dumps(x, ensure_ascii=False)
-    json_decode = lambda x: json.loads(unicode(x))
+
+json_encode = lambda x: json.dumps(x)
+json_decode = lambda x: json.loads(x)
 
 def readStringMsg():
     msg = ""
@@ -62,7 +61,7 @@ def sendToParent(s):
     print s
     print "end"
     sys.stdout.flush()
-    
+
 def sync():
     print "sync"
     sys.stdout.flush()
@@ -134,7 +133,7 @@ def initComponent():
     sendpid(heartbeatdir)
     return readenv()
 
-class Tuple:    
+class Tuple:
     def __init__(self, id, component, stream, task, values):
         self.id = id
         self.component = component
@@ -150,10 +149,10 @@ def __repr__(self):
 class Bolt:
     def initialize(self, stormconf, context):
         pass
-    
+
     def process(self, tuple):
         pass
-    
+
     def run(self):
         global MODE
         MODE = Bolt
@@ -164,15 +163,15 @@ def run(self):
                 tup = readTuple()
                 self.process(tup)
         except Exception, e:
-            log(traceback.format_exc(e))        
+            log(traceback.format_exc(e))
 
 class BasicBolt:
     def initialize(self, stormconf, context):
         pass
-    
+
     def process(self, tuple):
         pass
-    
+
     def run(self):
         global MODE
         MODE = Bolt
diff --git a/test/clj/backtype/storm/integration_test.clj b/test/clj/backtype/storm/integration_test.clj
