Building a common communication interface

I'm having trouble building an abstract interface for multiple communication types that have different settings to connect. I would like to be able to use a factory of some sort to instantiate one of the communication types (usb, serial, tcp) with their appropriate connection arguments without branching this off into a bunch of if statements everywhere to check the type and to use a single interface to control all the interaction with the specified communication type.

Some code below to help illustrate my problem:

public interface ITransport : IDisposable
{
    event EventHandler Connected;
    event EventHandler Disconnected;
    event EventHandler<byte[]> SentData;
    event EventHandler<byte[]> ReceivedData;
    event EventHandler<string> ErrorOccurred;

    event HostCertificateReceivedDelegate HostValidation;

    Task<bool> ConnectAsync(TransportConnectionArgs connectionArgs);
    Task<bool> SendAsync(byte[] buffer);

    void Disconnect();
}

public class TransportConnectionArgs
{
    public static readonly TransportConnectionArgs Empty = new TransportConnectionArgs();
    protected TransportConnectionArgs() { }
}

public class SshTransportConnectionArgs : TransportConnectionArgs
{
    public string Host { get; }
    public int Port { get; }

    public string Username { get; }
    public string Password { get; }

    public SshTransportConnectionArgs(string host, int port, string username = "root", string password = "")
    {
        Host = host;
        Port = port;
        Username = username;
        Password = password;
    }
}

public class SerialTransportConnectionArgs : TransportConnectionArgs
{
    ... does not need password but needs com port, baud, etc...
}

public class UsbTransportConnectionArgs : TransportConnectionArgs
{
    ... does not need address or port or serial connection settings
}

public sealed class SshDebugClient : ITransport
{
    public async Task<bool> ConnectAsync(SshTransportConnectionArgs connectionArgs)
    {
        ... access properties specific to a regular TCP connection
    }
}

public sealed class SerialDebugClient : ITransport
{
    public async Task<bool> ConnectAsync(SerialTransportConnectionArgs connectionArgs)
    {
        ... access properties specific to a regular serial/COM connection
    }
}

public sealed class UsbDebugClient : ITransport
{
    public async Task<bool> ConnectAsync(UsbTransportConnectionArgs connectionArgs)
    {
        ... access properties specific to a regular USB connection
    }
}