Under the hood of the Socket.IO + Express example

You might have seen one of the sneak preview videos on YouTube about a collaborative demo app using our scheduler with Ext JS and Sencha Touch. With our recent release of the first 2.2 beta version of Ext Scheduler, it’s now time to look under the hood of this demo application. When we were about to release the Touch Scheduler, we thought we should really put it to the test before releasing it. A few days later, we had written a collaborative app where changes are being broadcasted immediately when a task is modified. The live aspect of the application is not something you see often in web apps today but with tools like Express and Socket.IO it’s now very easy to accomplish. This post is quite long and contains lots of code, but here’s how we did it:

Setting up an Express web server

First of all, we had to setup a simple Express web server which runs on Node (you need at least Node 0.6+). Download Node.js here. We also need socket.io which you can find here. To avoid having to involve a database on the server, we simply created an in-memory array “DB” of tasks to serve as the database. It also has add/remove/update methods to support all CRUD operations. Below is the script which launches a simple Express instance on port 3000.

node_backend.js

///Module dependencies.
var http = require('http'),
    express = require('express'),
    app = module.exports = express.createServer(),
    fs = require('fs'),
    io = require('socket.io').listen(app);


//Server Configuration
app.configure(function(){
    app.use(express.bodyParser());
    app.use(express.methodOverride());
    app.use(express.static(__dirname + '/public'));
});

app.configure('development', function(){
    app.use(express.errorHandler({ dumpExceptions: true, showStack: true }));
});

app.get('/', function(req, res){
    res.sendfile(__dirname+'/public/index.html');
});

app.listen(3000);

//'DATABASE'
var DB = {
    eventsTable: [
        {Id: 1, ResourceId: 2, Name : 'Chase turkey', StartDate : "2010-12-09 08:00", EndDate : "2010-12-09 10:00", Done : true},
        {Id: 2, ResourceId: 1, Name : 'Stuff turkey', StartDate : "2010-12-09 10:00", EndDate : "2010-12-09 12:00", Done : true},
        {Id: 3, ResourceId: 3, Name : 'Cook turkey', StartDate : "2010-12-09 12:00", EndDate : "2010-12-09 15:00", Done : true},
        {Id: 4, ResourceId: 5, Name : 'Set table', StartDate : "2010-12-09 14:00", EndDate : "2010-12-09 16:00", Done : false},
        {Id: 5, ResourceId: 4, Name : 'Serve dinner', StartDate : "2010-12-09 16:00", EndDate : "2010-12-09 19:00", Done : false},
        {Id: 6, ResourceId: 6, Name : 'Hack on NodeJS', StartDate : "2010-12-09 16:00", EndDate : "2010-12-09 18:30", Done : false},
        {Id: 7, ResourceId: 7, Name : 'Clean up', StartDate : "2010-12-09 19:00", EndDate : "2010-12-09 20:00", Done : false},
        {Id: 8, ResourceId: 8, Name : 'Do laundry', StartDate : "2010-12-09 17:00", EndDate : "2010-12-09 19:00", Done : false}
    ],

    getEventsData: function(){
        return this.eventsTable;
    },

    // get record by ID
    getById: function(id){
        var table = this.getEventsData(),
            current;

        for(var i=0, l=table.length; i<l; i+=1){
            current = table[i];

            if(current.Id === id){
                return current;
            }
        }
        return null;
    },

    // update record
    update: function(record){
        var data       = record.data;
        var item       = this.getById(data.Id);

        for(var f in data){
            item[f] = data[f];
        }
    },

    // add records
    add: function(records){
        var table = this.getEventsData(),
            record,
            ID;

        for(var i=0, l=records.length; i<l; i+=1){
            record     = records[i];
            ID         = table[table.length-1].Id + 1;

            record.data.Id = ID;
            table.push(record.data);
        }
    },

    // remove records
    remove: function(ids){
        var table = this.getEventsData();

        ids.forEach(function(id) {
            item       = this.getById(id);
            table.splice(table.indexOf(item), 1);
        }, this);
    }
};

We then created a basic HTML index.html page which includes Ext JS, Ext Scheduler, Socket.IO and our application JS file (in the example there is also a touch version including Sencha Touch instead). Contents below:

index.html

<!--Ext and ux styles -->

<!--Scheduler styles-->

<!--Example styles -->

<!-- Socket IO-->
<script src="/socket.io/socket.io.js"></script>

<!--Ext 4 includes-->
<script src="https://localhost/extjs-4.2.0.265/ext-all-debug.js" type="text/javascript"></script>

<!--Scheduler files-->
<script src="https://localhost/ExtScheduler3.x/sch-all-debug.js" type="text/javascript"></script>

<!--Application files-->
<script src="/app.js" type="text/javascript"></script>

Now to the application bootstrap part, which means adding an app.js which should work for both Ext JS and Sencha Touch. This required a little bit of normalization code but nothing really difficult.
app.js
Ext.Loader.setConfig({
    enabled         : true,
    disableCaching  : true
});

Ext.require([
    'App.view.SchedulerGrid'
]);

// Limit the resolution, to avoid putting too much data on the wire.
Sch.preset.Manager.getPreset('hourAndDay').timeResolution.increment = 15;

Ext.application({
    name : 'App',

    viewport : {
        layout : {
            type : 'vbox',
            align : 'stretch'
        }
    },

    // Initialize application
    launch : function() {
        var field = new Ext.form.Text({
            fieldLabel : 'User ',
            height : 30,
            label : 'User ',
            value : 'John Doe',
            labelWidth: 115,
            listeners : {
                change : function(field, value) {
                    scheduler.userName = value;
                }
            }
        });

        var scheduler = Ext.create('App.view.SchedulerGrid', {
            title               : 'Node + socket.io example',
            startDate           : new Date(2010, 11, 9, 8),
            endDate             : new Date(2010, 11, 9, 20),
            flex                : 1,
            userName            : field.getValue()
        });

        var vp;

        if (Ext.versions.touch) {
            vp = Ext.Viewport;
        } else {
            // Ext JS
            vp = new Ext.Viewport(this.viewport);

            // Uncomment this to see what's happening in the EventStore
            //Ext.util.Observable.capture(scheduler.eventStore, function() { console.log(arguments); });

            scheduler.on('eventcontextmenu', this.onEventContextMenu, this);
            Ext.QuickTips.init();
        }

        vp.add([
            new Ext.form.Panel({
                region: 'north',
                hidden : Ext.os && Ext.os.is.phone,
                padding: 5,
                border : false,
                height : 55,
                items : field
            }),
            scheduler
        ]);
    },

    onEventContextMenu : function(scheduler, rec, e) {
        e.stopEvent();
        
        if (!scheduler.gCtx) {
            scheduler.gCtx = new Ext.menu.Menu({
                items : [
                    {
                        text : 'Delete event',
                        iconCls : 'icon-delete',
                        handler : function() {
                            scheduler.eventStore.remove(scheduler.gCtx.rec);
                        }
                    }  
                ]
            });
        }
        scheduler.gCtx.rec = rec;
        scheduler.gCtx.showAt(e.getXY());
    }    
});

This creates a simple Viewport and puts a scheduler inside it together with a form where you can enter your name. Let's move on to take a peek at the App.view.SchedulerGrid class.
UI classes
As with the app.js file, we had to make sure this class could be used regardless of the underlying Sencha framework. For this view class, we had to use the constructor instead of relying on the initComponent hook since this hook doesn't exist in Sencha Touch (instead there is an init hook). By relying on the constructor, we saved us a bit of normalization code.
App.view.SchedulerGrid
Ext.define('App.view.SchedulerGrid', {
    extend              : 'Sch.panel.SchedulerGrid',

    requires            : ['App.store.EventStore', 'App.store.ResourceStore'],

    userName            : null,
    draggingRecord      : null,
    socketHost          : null,
    rowHeight           : 75,
    barMargin           : 10,
    eventBarTextField   : 'Name',
    viewPreset          : 'hourAndDay',
    eventBodyTemplate   : '
{Name}

{BlockedBy}

', constructor : function() { var me = this; //create a WebSocket and connect to the server running at host domain var socket = me.socket = io.connect(me.socketHost); Ext.apply(me, { viewConfig : { onEventUpdate: function (store, model, operation) { // Skip local paints of the record currently being dragged if (model !== me.draggingRecord) { this.horizontal.onEventUpdate(store, model, operation); } } }, columns : [ { header : 'Name', width : 120, dataIndex : 'Name', sortable : true} ], eventRenderer : function(event, resource, tplData) { tplData.cls = ''; if (event.data.Done) { tplData.cls += ' sch-event-done '; } if (event.data.Blocked) { tplData.cls += ' sch-event-blocked '; if (event === me.draggingRecord) { tplData.cls += ' x-hidden '; } }; return event.data; }, resourceStore : new App.store.ResourceStore({ /* Extra configs here */}), eventStore : new App.store.EventStore({ socket : socket }) }); this.callParent(arguments); // Change default drag drop behavior to update the dragged record 'live' me.on({ eventdragstart : me.onDragStart, eventdrag : me.onEventDrag, aftereventdrop : me.onDragEnd, scope : me }); }, onEventCreated : function(record) { record.set('Name', 'New task'); }, // Block a record when it is being dragged onDragStart : function(view, records) { var rec = records[0]; this.draggingRecord = rec; rec.block(this.userName); }, // Update underlying record as it is moved around in the schedule onEventDrag : function(sch, draggedRecords, startDate, newResource) { if (newResource && startDate) { var task = draggedRecords[0]; task.beginEdit(); task.setStartDate(startDate, true); task.assign(newResource); task.endEdit(); } }, // Unblock a record after dragging it onDragEnd : function(view, records) { this.draggingRecord = null; records[0].unblock(); } });

There are basically just two special things about the scheduler above: When a drag operation starts, the dragged Model is marked as blocked and as the cursor is moved, the model is continuously updated until the drag operation is finished. Since we don't want our own 'local' view to repaint itself on such model updates, we override the onEventUpdate method to prevent it.
Data classes
So far we have only presented the UI and the application classes. Let's move on to where the real magic happens. First, let's look at the underlying App.model.CustomEvent, its sources can be seen below. To have this class support both Sencha frameworks, we needed a bit of normalization code since ST uses its config system to define class members.
App.model.CustomEvent
(function() {
    var fields = [
        { name  : 'Blocked' },
        { name  : 'BlockedBy' },
        { name  : 'Done', type : 'boolean'}
    ];

    Ext.define('App.model.CustomEvent', {
        extend      : 'Sch.model.Event',
        fields      : fields,

        // Sencha Touch
        config      : {
            fields  : fields
        },

        block       : function(userName) {
            this.set({
                Blocked     : true,
                BlockedBy   : userName
            })
        },

        unblock     : function() {
            this.set({
                Blocked     : false,
                BlockedBy   : null
            });
        }
    });
}());

This model is consumed by the App.store.EventStore which is just a plain store consuming a special mixin we wrote. It defines the model to use and also initializes the socket its bound to.
App.store.EventStore
Ext.define('App.store.EventStore', {
    extend : "Sch.data.EventStore",

    requires  : [
        'App.model.CustomEvent'
    ],

    config : Ext.versions.touch ? {
        socket  : null,
        model   : 'App.model.CustomEvent'
    } : null,

    model   : 'App.model.CustomEvent',
    mixins  : [
        'App.store.mixin.SocketIO'
    ],

    proxy: {
        type: 'memory',
        reader: {
            type: 'json'
        }
    },

    constructor : function() {
        this.callParent(arguments);
        this.initSocket();
    }
});

Socket.IO - client side
If you have worked with regular Sencha data stores before, you'll note we are doing things a bit different here. We're not using any of the load/save capabilities of the data package. We're instead letting socket.io handle all the CRUD traffic to and from the server. The contract for this mixin class can be summarized as this statement:
Observe the socket to know what others are doing, and let the others know what I'm doing
The socket API allows you to observe it using the on method, and you can use the emit to broadcast local changes.
App.store.mixin.SocketIO
Ext.define('App.store.mixin.SocketIO', {
    socket : null,

    getSocket : function () {
        return this.socket;
    },

    initSocket : function () {

        var that = this;
        var socket = this.getSocket();

        socket.on('server-doInitialLoad', function (data) {
            that.onInitialLoad(data);
        });
        socket.on('server-doUpdate', function (data) {
            that.onRemoteUpdate(data);
        });
        socket.on('server-doAdd', function (data) {
            that.onRemoteAdd(data);
        });
        socket.on('server-syncId', function (data) {
            that.onRemoteSyncId(data);
        });
        socket.on('server-doRemove', function (data) {
            that.onRemoteRemove(data);
        });

        this.myListeners = {
            add             : this.onLocalAdd,
            update          : this.onLocalUpdate,
            remove          : this.onLocalRemove,
            addrecords      : this.onLocalAdd,
            updaterecord    : this.onLocalUpdate,
            removerecords   : this.onLocalRemove
        };

        this.addMyListeners();

        //Load initial data to Store from Server
        this.doInitialLoad();
    },

    addMyListeners : function () {
        //Add event listeners to store operations
        this.on(this.myListeners);
    },

    removeMyListeners : function () {
        //Add event listeners to store operations
        this.un(this.myListeners);
    },

    /**
     * Emit event to server in order to receive initial data for store from the DB.
     */
    doInitialLoad : function () {
        this.getSocket().emit('client-doInitialLoad');
    },

    /* BEGIN REMOTE LISTENER METHODS */

    /**
     * New records were added remotely, add them to our local client store
     */
    onRemoteAdd : function (data) {
        var records = data.records,
            record,
            current,
            model = this.getModel();

        this.removeMyListeners();

        for (var i = 0, l = records.length; i < l; i += 1) {
            current = records[i].data;

            //change dates from JSON form to Date
            current.startDate = new Date(current.StartDate);
            current.endDate = new Date(current.EndDate);

            // Work around a bug in ST when adding new records (internalId not set correctly)
            this.add(new model(current, current.Id));
        }

        this.addMyListeners();
    },

    onRemoteSyncId : function (data) {
        var records = data.records,
            model = this.getModel();

        this.removeMyListeners();

        Ext.Array.each(records, function(updatedRecord) {
            var internalId = updatedRecord.internalId;

            this.each(function (rec, idx) {
                if (rec.internalId == internalId) {
                    this.remove(rec);
                    this.add(new model(updatedRecord.data, updatedRecord.data.Id))
                    return false;
                }
            }, this);
        }, this);

        this.addMyListeners();
    },

    /**
     * Records were updated remotely, update them in our local client store
     */
    onRemoteUpdate : function (data) {
        var localRecord;

        // Only one record updated at a time
        data = data.record.data;

        this.removeMyListeners();

        localRecord = this.getById(data.Id);
        if (localRecord) {
            data.StartDate && (data.StartDate = new Date(data.StartDate));
            data.EndDate && (data.EndDate = new Date(data.EndDate));

            localRecord.set(data);
        }

        this.addMyListeners();
    },

    /**
     * Records were removed remotely, remove them from our local client store
     */
    onRemoteRemove : function (data) {
        var ids = data.data,
            record,
            current;

        this.removeMyListeners();

        for (var i = 0, l = ids.length; i < l; i += 1) {
            current = ids[i];
            record = this.getById(current);

            this.remove(record);
        }

        this.addMyListeners();
    },

    /**
     * Initial data loaded from server.
     */
    onInitialLoad : function (data) {
        var data = data.data;
        (this.loadData || this.setData).call(this, data);
    },

    /* EOF REMOTE LISTENER METHODS */


    /* BEGIN LOCAL STORE LISTENER METHODS */

    /**
     * On adding records to client store, send event to server and add items to DB.
     */
    onLocalAdd : function (store, records, index, opts) {
        var recordsData = [];
        records = records.length ? records : [records];

        for (var i = 0, l = records.length; i < l; i += 1) {
            records[i].data.Name = 'New Assignment';

            recordsData.push({
                data : records[i].data,
                internalId : records[i].internalId
            });
        }

        this.getSocket().emit('client-doAdd', { records : recordsData });
    },

    /**
     * On updating records in client store, send event to server and update items in DB.
     */
    onLocalUpdate : function (store, record) {
        var data = { Id : record.getId() };

        for (var prop in record.previous) {
            data[prop] = record.data[prop];
        }

        this.getSocket().emit('client-doUpdate', { record : { data : data } });
    },

    /**
     * On adding removing records from client store, send event to server and remove items from DB.
     */
    onLocalRemove : function (store, records, index, opts) {
        records = records.length ? records : [records];
        var ids = Ext.Array.map(records, function (rec) {
            return rec.get('Id');
        });

        this.getSocket().emit('client-doRemove', { ids : ids });
    }

    /* EOF LOCAL STORE LISTENER METHODS */
});

If you read the code above, you'll also notice a few framework normalization snippets, but all in all it's a trivial data mixin class. The final piece of this application is the socket.io integration on the server side.
Socket.IO - server side
This is a quite straight forward piece of code. Once the socket connection is available we observe it for actions invoked on the client side (load/update/add/remove). Using the emit method of the broadcast property of the socket, we can reach all connected clients.
node_backend.js
// WEBSOCKETS COMMUNICATION
io.sockets.on('connection', function (socket) {

    //Load initial data to client Store
    socket.on('client-doInitialLoad', function(data){
        socket.emit('server-doInitialLoad', { data : DB.getEventsData()});
    });

    //Update records in DB and inform other clients about the change
    socket.on('client-doUpdate', function(data){
        var record   = data.record;

        DB.update(record);

        socket.broadcast.emit('server-doUpdate', data);
    });

    //Add record to DB and inform other clients about the change
    socket.on('client-doAdd', function(data){
        var records   = data.records;

        DB.add(records);

        socket.broadcast.emit('server-doAdd', { records : records });

        //Sync ID of new record with client Store
        socket.emit('server-syncId', { records : records });
    });

    //Remove record from DB and inform other clients about the change
    socket.on('client-doRemove', function(data){
        var ids   = data.ids;

        DB.remove(ids);

        socket.broadcast.emit('server-doRemove', { data : data.ids });
    });
});

Styling
Finally, to visualize that someone else is moving a task bar on your screen, we placed a hand cursor for tasks that are in the 'Blocked' state. Next to the cursor, we also show the name of the user moving the task. This gives any observing users an extra clue about what is going on.

Summing up...
By combining our products with Ext JS, Sencha Touch, Express and Socket.IO we were able to write a cool collaborative application with realtime updates. We were also able to reuse the application code for both Sencha frameworks with only a very tiny amount of normalization code. If you like this example and find it useful (or if you have suggestions of how it can be improved), please let us know in the comments. To try it out, you need to download the Ext Scheduler 2.2 version and look in the '/examples/nodejs' folder. There is an 'index.html' file for desktop and an 'index_touch.html' for touch devices.