var check_count=0;
function checkbox_limit()
{
    for(i = 0;i < document.form_name.length;i++)
    {
       if(document.form_name.elements[i].type!="checkbox") continue;
       if(document.form_name.elements[i].checked)
       {
           if(check_count==4)
           {
               alert("随便");
               document.form_name.elements[i].checked=false;
               return false;
            }
           else
           check_count++;
       }
    }
}...
<input type="checkbox" name="checkbox_name[]" value="" onclick="checkbox_limit()">