First you need to specify the layer with getLayer() or findLayers() and then use the setParameter.
function subLayer1MuteChanged()
local layer = this.parent:findLayers(true, "SubLayer1_1")[1]
if layer then
layer:setParameter("LayerMute", Sub1Mute)
end
end
defineParameter("Sub1Mute", nil, false, subLayer1MuteChanged)