From 339d65dff87dc173cc5b0800140a353ba5181c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 24 Nov 2022 12:15:10 +0100 Subject: [PATCH 01/19] Handle the raw ICE connection state in the views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than just providing a coarse "connected" or "not connected" value now the views receive the raw ICE connection state. Combined with other properties this will make possible to show a finer grained status (like done in the WebUI), although for now just "connected" or "not connected" is still shown as before. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 55 ++++--------------- .../talk/adapters/ParticipantDisplayItem.java | 14 +++-- 2 files changed, 20 insertions(+), 49 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 81c1030d24..abff929256 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -2072,7 +2072,10 @@ public void onMessageEvent(ConfigurationChangeEvent configurationChangeEvent) { updateSelfVideoViewPosition(); } - private void updateSelfVideoViewConnected(boolean connected) { + private void updateSelfVideoViewIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { + boolean connected = iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || + iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; + // FIXME In voice only calls there is no video view, so the progress bar would appear floating in the middle of // nowhere. However, a way to signal that the local participant is not connected to the HPB is still need in // that case. @@ -2133,28 +2136,6 @@ public void onMessageEvent(ProximitySensorEvent proximitySensorEvent) { } } - private void handlePeerConnected(String sessionId, String videoStreamType) { - String participantDisplayItemId = sessionId + "-" + videoStreamType; - - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - updateSelfVideoViewConnected(true); - } else if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setConnected(true); - participantsAdapter.notifyDataSetChanged(); - } - } - - private void handlePeerDisconnected(String sessionId, String videoStreamType) { - String participantDisplayItemId = sessionId + "-" + videoStreamType; - - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - updateSelfVideoViewConnected(false); - } else if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setConnected(false); - participantsAdapter.notifyDataSetChanged(); - } - } - private void startSendingNick() { DataChannelMessage dataChannelMessage = new DataChannelMessage(); dataChannelMessage.setType("nickChanged"); @@ -2211,11 +2192,9 @@ private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(session, videoStreamType); - boolean connected = false; + PeerConnection.IceConnectionState iceConnectionState = null; if (peerConnectionWrapper != null) { - PeerConnection.IceConnectionState iceConnectionState = peerConnectionWrapper.getPeerConnection().iceConnectionState(); - connected = iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || - iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; + iceConnectionState = peerConnectionWrapper.getPeerConnection().iceConnectionState(); } String nick; @@ -2232,7 +2211,7 @@ private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, ParticipantDisplayItem participantDisplayItem = new ParticipantDisplayItem(baseUrl, userId, session, - connected, + iceConnectionState, nick, defaultGuestNick, mediaStream, @@ -2692,19 +2671,11 @@ private void handleStream(MediaStream mediaStream) { @Override public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { runOnUiThread(() -> { - if (iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || - iceConnectionState == PeerConnection.IceConnectionState.COMPLETED) { - handlePeerConnected(sessionId, videoStreamType); - - return; - } - - if (iceConnectionState == PeerConnection.IceConnectionState.DISCONNECTED || - iceConnectionState == PeerConnection.IceConnectionState.NEW || - iceConnectionState == PeerConnection.IceConnectionState.CHECKING) { - handlePeerDisconnected(sessionId, videoStreamType); - - return; + if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { + updateSelfVideoViewIceConnectionState(iceConnectionState); + } else if (participantDisplayItems.get(participantDisplayItemId) != null) { + participantDisplayItems.get(participantDisplayItemId).setIceConnectionState(iceConnectionState); + participantsAdapter.notifyDataSetChanged(); } if (iceConnectionState == PeerConnection.IceConnectionState.CLOSED) { @@ -2718,8 +2689,6 @@ public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceCon setCallState(CallStatus.PUBLISHER_FAILED); webSocketClient.clearResumeId(); hangup(false); - } else { - handlePeerDisconnected(sessionId, videoStreamType); } return; diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 34fa963acd..63a0d3ef53 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -6,12 +6,13 @@ import org.webrtc.EglBase; import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; public class ParticipantDisplayItem { private String baseUrl; private String userId; private String session; - private boolean connected; + private PeerConnection.IceConnectionState iceConnectionState; private String nick; private final String defaultGuestNick; private String urlForAvatar; @@ -21,11 +22,11 @@ public class ParticipantDisplayItem { private EglBase rootEglBase; private boolean isAudioEnabled; - public ParticipantDisplayItem(String baseUrl, String userId, String session, boolean connected, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { + public ParticipantDisplayItem(String baseUrl, String userId, String session, PeerConnection.IceConnectionState iceConnectionState, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { this.baseUrl = baseUrl; this.userId = userId; this.session = session; - this.connected = connected; + this.iceConnectionState = iceConnectionState; this.nick = nick; this.defaultGuestNick = defaultGuestNick; this.mediaStream = mediaStream; @@ -55,11 +56,12 @@ public void setSession(String session) { } public boolean isConnected() { - return connected; + return iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || + iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; } - public void setConnected(boolean connected) { - this.connected = connected; + public void setIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { + this.iceConnectionState = iceConnectionState; } public String getNick() { From e887fde2a349e33bb5f7fc195ccbf001c6d07b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 13:12:42 +0100 Subject: [PATCH 02/19] Remove unused getters and setters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note that the session ID, user ID and the stream type attributes are still kept, as they can be useful to identify the instance when debugging. Signed-off-by: Daniel Calviño Sánchez --- .../talk/adapters/ParticipantDisplayItem.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 63a0d3ef53..fa980ad5d5 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -37,24 +37,12 @@ public ParticipantDisplayItem(String baseUrl, String userId, String session, Pee this.updateUrlForAvatar(); } - public String getUserId() { - return userId; - } - public void setUserId(String userId) { this.userId = userId; this.updateUrlForAvatar(); } - public String getSession() { - return session; - } - - public void setSession(String session) { - this.session = session; - } - public boolean isConnected() { return iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; @@ -98,14 +86,6 @@ public void setMediaStream(MediaStream mediaStream) { this.mediaStream = mediaStream; } - public String getStreamType() { - return streamType; - } - - public void setStreamType(String streamType) { - this.streamType = streamType; - } - public boolean isStreamEnabled() { return streamEnabled; } @@ -118,10 +98,6 @@ public EglBase getRootEglBase() { return rootEglBase; } - public void setRootEglBase(EglBase rootEglBase) { - this.rootEglBase = rootEglBase; - } - public boolean isAudioEnabled() { return isAudioEnabled; } From 5fe9154c9ae3cedbf985ba242b780484fd8e2f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 13:13:27 +0100 Subject: [PATCH 03/19] Declare attributes set just once in constructor as "final" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../nextcloud/talk/adapters/ParticipantDisplayItem.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index fa980ad5d5..b9744a8f8d 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -9,17 +9,17 @@ import org.webrtc.PeerConnection; public class ParticipantDisplayItem { - private String baseUrl; + private final String baseUrl; private String userId; - private String session; + private final String session; private PeerConnection.IceConnectionState iceConnectionState; private String nick; private final String defaultGuestNick; private String urlForAvatar; private MediaStream mediaStream; - private String streamType; + private final String streamType; private boolean streamEnabled; - private EglBase rootEglBase; + private final EglBase rootEglBase; private boolean isAudioEnabled; public ParticipantDisplayItem(String baseUrl, String userId, String session, PeerConnection.IceConnectionState iceConnectionState, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { From d67b04dff8aec30ab56dfaf93db28670b304802e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 13:14:46 +0100 Subject: [PATCH 04/19] Reorder attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generic final attributes first, followed by object specific final attributes and then other object attributes. Signed-off-by: Daniel Calviño Sánchez --- .../talk/adapters/ParticipantDisplayItem.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index b9744a8f8d..1ababf4172 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -10,16 +10,18 @@ public class ParticipantDisplayItem { private final String baseUrl; - private String userId; + private final String defaultGuestNick; + private final EglBase rootEglBase; + private final String session; + private final String streamType; + + private String userId; private PeerConnection.IceConnectionState iceConnectionState; private String nick; - private final String defaultGuestNick; private String urlForAvatar; private MediaStream mediaStream; - private final String streamType; private boolean streamEnabled; - private final EglBase rootEglBase; private boolean isAudioEnabled; public ParticipantDisplayItem(String baseUrl, String userId, String session, PeerConnection.IceConnectionState iceConnectionState, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { From 8a316d94f545f74deb78487d43f758fe81aa4fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 20:05:21 +0100 Subject: [PATCH 05/19] Notify that data set changed automatically when display item changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of explicitly calling "notifyDataSetChanged" after setting values on a ParticipantDisplayItem now the adapter observes all its items and calls "notifyDataSetChanged" automatically when any of them changes. Although this adds some boilerplate code it will make possible to update the ParticipantDisplayItems and automatically propagate the changes to the adapter when a model changes, rather than having to explicitly do it from the CallActivity. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 23 ++------ .../talk/adapters/ParticipantDisplayItem.java | 27 +++++++++ .../ParticipantDisplayItemNotifier.java | 55 +++++++++++++++++++ .../talk/adapters/ParticipantsAdapter.java | 11 ++++ 4 files changed, 97 insertions(+), 19 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItemNotifier.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index abff929256..1fd0d2c997 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -740,6 +740,10 @@ public void onGlobalLayout() { } }); + if (participantsAdapter != null) { + participantsAdapter.destroy(); + } + participantsAdapter = new ParticipantsAdapter( this, participantDisplayItems, @@ -1853,17 +1857,11 @@ private void processUsersInRoom(List participants) { String userId = userIdsBySessionId.get(sessionId); if (userId != null) { runOnUiThread(() -> { - boolean notifyDataSetChanged = false; if (participantDisplayItems.get(sessionId + "-video") != null) { participantDisplayItems.get(sessionId + "-video").setUserId(userId); - notifyDataSetChanged = true; } if (participantDisplayItems.get(sessionId + "-screen") != null) { participantDisplayItems.get(sessionId + "-screen").setUserId(userId); - notifyDataSetChanged = true; - } - if (notifyDataSetChanged) { - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2527,17 +2525,11 @@ private OfferAnswerNickProvider(String sessionId) { private void onOfferOrAnswer(String nick) { this.nick = nick; - boolean notifyDataSetChanged = false; if (participantDisplayItems.get(sessionId + "-video") != null) { participantDisplayItems.get(sessionId + "-video").setNick(nick); - notifyDataSetChanged = true; } if (participantDisplayItems.get(sessionId + "-screen") != null) { participantDisplayItems.get(sessionId + "-screen").setNick(nick); - notifyDataSetChanged = true; - } - if (notifyDataSetChanged) { - participantsAdapter.notifyDataSetChanged(); } } @@ -2582,7 +2574,6 @@ public void onAudioOn() { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(true); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2592,7 +2583,6 @@ public void onAudioOff() { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(false); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2602,7 +2592,6 @@ public void onVideoOn() { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(true); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2612,7 +2601,6 @@ public void onVideoOff() { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(false); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2622,7 +2610,6 @@ public void onNickChanged(String nick) { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setNick(nick); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2664,7 +2651,6 @@ private void handleStream(MediaStream mediaStream) { ParticipantDisplayItem participantDisplayItem = participantDisplayItems.get(participantDisplayItemId); participantDisplayItem.setMediaStream(mediaStream); participantDisplayItem.setStreamEnabled(hasAtLeastOneVideoStream); - participantsAdapter.notifyDataSetChanged(); }); } @@ -2675,7 +2661,6 @@ public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceCon updateSelfVideoViewIceConnectionState(iceConnectionState); } else if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setIceConnectionState(iceConnectionState); - participantsAdapter.notifyDataSetChanged(); } if (iceConnectionState == PeerConnection.IceConnectionState.CLOSED) { diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 1ababf4172..746f217103 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -9,6 +9,13 @@ import org.webrtc.PeerConnection; public class ParticipantDisplayItem { + + public interface Observer { + void onChange(); + } + + private final ParticipantDisplayItemNotifier participantDisplayItemNotifier = new ParticipantDisplayItemNotifier(); + private final String baseUrl; private final String defaultGuestNick; private final EglBase rootEglBase; @@ -43,6 +50,8 @@ public void setUserId(String userId) { this.userId = userId; this.updateUrlForAvatar(); + + participantDisplayItemNotifier.notifyChange(); } public boolean isConnected() { @@ -52,6 +61,8 @@ public boolean isConnected() { public void setIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { this.iceConnectionState = iceConnectionState; + + participantDisplayItemNotifier.notifyChange(); } public String getNick() { @@ -66,6 +77,8 @@ public void setNick(String nick) { this.nick = nick; this.updateUrlForAvatar(); + + participantDisplayItemNotifier.notifyChange(); } public String getUrlForAvatar() { @@ -86,6 +99,8 @@ public MediaStream getMediaStream() { public void setMediaStream(MediaStream mediaStream) { this.mediaStream = mediaStream; + + participantDisplayItemNotifier.notifyChange(); } public boolean isStreamEnabled() { @@ -94,6 +109,8 @@ public boolean isStreamEnabled() { public void setStreamEnabled(boolean streamEnabled) { this.streamEnabled = streamEnabled; + + participantDisplayItemNotifier.notifyChange(); } public EglBase getRootEglBase() { @@ -106,6 +123,16 @@ public boolean isAudioEnabled() { public void setAudioEnabled(boolean audioEnabled) { isAudioEnabled = audioEnabled; + + participantDisplayItemNotifier.notifyChange(); + } + + public void addObserver(Observer observer) { + participantDisplayItemNotifier.addObserver(observer); + } + + public void removeObserver(Observer observer) { + participantDisplayItemNotifier.removeObserver(observer); } @Override diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItemNotifier.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItemNotifier.java new file mode 100644 index 0000000000..239b9f85b9 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItemNotifier.java @@ -0,0 +1,55 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.adapters; + +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Helper class to register and notify ParticipantDisplayItem.Observers. + * + * This class is only meant for internal use by ParticipantDisplayItem; observers must register themselves against a + * ParticipantDisplayItem rather than against a ParticipantDisplayItemNotifier. + */ +class ParticipantDisplayItemNotifier { + + private final Set participantDisplayItemObservers = new LinkedHashSet<>(); + + public synchronized void addObserver(ParticipantDisplayItem.Observer observer) { + if (observer == null) { + throw new IllegalArgumentException("ParticipantDisplayItem.Observer can not be null"); + } + + participantDisplayItemObservers.add(observer); + } + + public synchronized void removeObserver(ParticipantDisplayItem.Observer observer) { + participantDisplayItemObservers.remove(observer); + } + + public synchronized void notifyChange() { + for (ParticipantDisplayItem.Observer observer : new ArrayList<>(participantDisplayItemObservers)) { + observer.onChange(); + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java index b8b5cc60b5..85e24d50d6 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java @@ -29,6 +29,8 @@ public class ParticipantsAdapter extends BaseAdapter { private static final String TAG = "ParticipantsAdapter"; + private final ParticipantDisplayItem.Observer participantDisplayItemObserver = this::notifyDataSetChanged; + private final Context mContext; private final ArrayList participantDisplayItems; private final RelativeLayout gridViewWrapper; @@ -50,8 +52,17 @@ public ParticipantsAdapter(Context mContext, this.participantDisplayItems = new ArrayList<>(); this.participantDisplayItems.addAll(participantDisplayItems.values()); + + for (ParticipantDisplayItem participantDisplayItem : this.participantDisplayItems) { + participantDisplayItem.addObserver(participantDisplayItemObserver); + } } + public void destroy() { + for (ParticipantDisplayItem participantDisplayItem : participantDisplayItems) { + participantDisplayItem.removeObserver(participantDisplayItemObserver); + } + } @Override public int getCount() { From d72648379eefc2c9b2990e8cbfd65ce25400ba78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 21:49:47 +0100 Subject: [PATCH 06/19] Add model for (remote) call participants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clients that modify the model would define the variables using the mutable subclass, while clients that only need to access the model are expected to use the read-only base class. The read-only class provides an observer; as it is expected that the model will be modified from background threads but observed from the main thread the observer can be registered along a handler to be notified on its thread, independently of on which thread the values were set. Currently there does not seem to be a need to observe each value on its own, so the observer is notified in a coarse way when any value changes. Signed-off-by: Daniel Calviño Sánchez --- .../talk/call/CallParticipantModel.java | 164 ++++++++++++++++++ .../call/CallParticipantModelNotifier.java | 86 +++++++++ .../call/MutableCallParticipantModel.java | 67 +++++++ 3 files changed, 317 insertions(+) create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java create mode 100644 app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java new file mode 100644 index 0000000000..8c39478246 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java @@ -0,0 +1,164 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import android.os.Handler; + +import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; + +import java.util.Objects; + +/** + * Read-only data model for (remote) call participants. + * + * The received audio and video are available only if the participant is sending them and also has them enabled. + * Before a connection is established it is not known whether audio and video are available or not, so null is returned + * in that case (therefore it should not be autoboxed to a plain boolean without checking that). + * + * Audio and video in screen shares, on the other hand, are always seen as available. + * + * Clients of the model can observe it with CallParticipantModel.Observer to be notified when any value changes. + * Getters called after receiving a notification are guaranteed to provide at least the value that triggered the + * notification, but it may return even a more up to date one (so getting the value again on the following + * notification may return the same value as before). + */ +public class CallParticipantModel { + + public interface Observer { + void onChange(); + } + + protected class Data { + + private T value; + + public T getValue() { + return value; + } + + public void setValue(T value) { + if (Objects.equals(this.value, value)) { + return; + } + + this.value = value; + + callParticipantModelNotifier.notifyChange(); + } + } + + private final CallParticipantModelNotifier callParticipantModelNotifier = new CallParticipantModelNotifier(); + + protected final String sessionId; + + protected Data userId; + protected Data nick; + + protected Data iceConnectionState; + protected Data mediaStream; + protected Data audioAvailable; + protected Data videoAvailable; + + protected Data screenIceConnectionState; + protected Data screenMediaStream; + + public CallParticipantModel(String sessionId) { + this.sessionId = sessionId; + + this.userId = new Data<>(); + this.nick = new Data<>(); + + this.iceConnectionState = new Data<>(); + this.mediaStream = new Data<>(); + this.audioAvailable = new Data<>(); + this.videoAvailable = new Data<>(); + + this.screenIceConnectionState = new Data<>(); + this.screenMediaStream = new Data<>(); + } + + public String getSessionId() { + return sessionId; + } + + public String getUserId() { + return userId.getValue(); + } + + public String getNick() { + return nick.getValue(); + } + + public PeerConnection.IceConnectionState getIceConnectionState() { + return iceConnectionState.getValue(); + } + + public MediaStream getMediaStream() { + return mediaStream.getValue(); + } + + public Boolean isAudioAvailable() { + return audioAvailable.getValue(); + } + + public Boolean isVideoAvailable() { + return videoAvailable.getValue(); + } + + public PeerConnection.IceConnectionState getScreenIceConnectionState() { + return screenIceConnectionState.getValue(); + } + + public MediaStream getScreenMediaStream() { + return screenMediaStream.getValue(); + } + + /** + * Adds an Observer to be notified when any value changes. + * + * @param observer the Observer + * @see CallParticipantModel#addObserver(Observer, Handler) + */ + public void addObserver(Observer observer) { + addObserver(observer, null); + } + + /** + * Adds an observer to be notified when any value changes. + * + * The observer will be notified on the thread associated to the given handler. If no handler is given the + * observer will be immediately notified on the same thread that changed the value; the observer will be + * immediately notified too if the thread of the handler is the same thread that changed the value. + * + * An observer is expected to be added only once. If the same observer is added again it will be notified just + * once on the thread of the last handler. + * + * @param observer the Observer + * @param handler a Handler for the thread to be notified on + */ + public void addObserver(Observer observer, Handler handler) { + callParticipantModelNotifier.addObserver(observer, handler); + } + + public void removeObserver(Observer observer) { + callParticipantModelNotifier.removeObserver(observer); + } +} diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java new file mode 100644 index 0000000000..ddf30c1d7b --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java @@ -0,0 +1,86 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import android.os.Handler; +import android.os.Looper; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Helper class to register and notify CallParticipantModel.Observers. + * + * This class is only meant for internal use by CallParticipantModel; observers must register themselves against a + * CallParticipantModel rather than against a CallParticipantModelNotifier. + */ +class CallParticipantModelNotifier { + + /** + * Helper class to associate a CallParticipantModel.Observer with a Handler. + */ + private static class CallParticipantModelObserverOn { + public final CallParticipantModel.Observer observer; + public final Handler handler; + + private CallParticipantModelObserverOn(CallParticipantModel.Observer observer, Handler handler) { + this.observer = observer; + this.handler = handler; + } + } + + private final List callParticipantModelObserversOn = new ArrayList<>(); + + public synchronized void addObserver(CallParticipantModel.Observer observer, Handler handler) { + if (observer == null) { + throw new IllegalArgumentException("CallParticipantModel.Observer can not be null"); + } + + removeObserver(observer); + + callParticipantModelObserversOn.add(new CallParticipantModelObserverOn(observer, handler)); + } + + public synchronized void removeObserver(CallParticipantModel.Observer observer) { + Iterator it = callParticipantModelObserversOn.iterator(); + while (it.hasNext()) { + CallParticipantModelObserverOn observerOn = it.next(); + + if (observerOn.observer == observer) { + it.remove(); + + return; + } + } + } + + public synchronized void notifyChange() { + for (CallParticipantModelObserverOn observerOn : new ArrayList<>(callParticipantModelObserversOn)) { + if (observerOn.handler == null || observerOn.handler.getLooper() == Looper.myLooper()) { + observerOn.observer.onChange(); + } else { + observerOn.handler.post(() -> { + observerOn.observer.onChange(); + }); + } + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java b/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java new file mode 100644 index 0000000000..4023bd2962 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java @@ -0,0 +1,67 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; + +/** + * Mutable data model for (remote) call participants. + * + * There is no synchronization when setting the values; if needed, it should be handled by the clients of the model. + */ +public class MutableCallParticipantModel extends CallParticipantModel { + + public MutableCallParticipantModel(String sessionId) { + super(sessionId); + } + + public void setUserId(String userId) { + this.userId.setValue(userId); + } + + public void setNick(String nick) { + this.nick.setValue(nick); + } + + public void setIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { + this.iceConnectionState.setValue(iceConnectionState); + } + + public void setMediaStream(MediaStream mediaStream) { + this.mediaStream.setValue(mediaStream); + } + + public void setAudioAvailable(Boolean audioAvailable) { + this.audioAvailable.setValue(audioAvailable); + } + + public void setVideoAvailable(Boolean videoAvailable) { + this.videoAvailable.setValue(videoAvailable); + } + + public void setScreenIceConnectionState(PeerConnection.IceConnectionState screenIceConnectionState) { + this.screenIceConnectionState.setValue(screenIceConnectionState); + } + + public void setScreenMediaStream(MediaStream screenMediaStream) { + this.screenMediaStream.setValue(screenMediaStream); + } +} From 18f21c4f482a573b32173966d749c89e2c144316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 22:07:05 +0100 Subject: [PATCH 07/19] Update ParticipantDisplayItem from CallParticipantModel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of explicitly setting the values on the ParticipantDisplayItems now the values are set on the CallParticipantModels, and the items are automatically updated from their model when they change. Different items are still used for the audio/video and screen shares of the same participant, so the type is used to select from which properties of the model is the item updated. As the model may be updated from background threads it is explicitly observed by the items from the main thread using a Handler shared by all the items. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 189 +++++++++--------- .../talk/adapters/ParticipantDisplayItem.java | 101 +++++----- 2 files changed, 143 insertions(+), 147 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 1fd0d2c997..321b8257bb 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -60,6 +60,8 @@ import com.nextcloud.talk.adapters.ParticipantsAdapter; import com.nextcloud.talk.api.NcApi; import com.nextcloud.talk.application.NextcloudTalkApplication; +import com.nextcloud.talk.call.CallParticipantModel; +import com.nextcloud.talk.call.MutableCallParticipantModel; import com.nextcloud.talk.data.user.model.User; import com.nextcloud.talk.databinding.CallActivityBinding; import com.nextcloud.talk.events.ConfigurationChangeEvent; @@ -231,7 +233,6 @@ public class CallActivity extends CallBaseActivity { private MediaStream localStream; private String credentials; private List peerConnectionWrapperList = new ArrayList<>(); - private Map userIdsBySessionId = new HashMap<>(); private boolean videoOn = false; private boolean microphoneOn = false; @@ -267,6 +268,8 @@ public class CallActivity extends CallBaseActivity { private Map peerConnectionObservers = new HashMap<>(); + private Map callParticipantModels = new HashMap<>(); + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { @Override @@ -382,6 +385,7 @@ public void onCreate(Bundle savedInstanceState) { requestBluetoothPermission(); } basicInitialization(); + callParticipantModels = new HashMap<>(); participantDisplayItems = new HashMap<>(); initViews(); if (!isConnectionEstablished()) { @@ -1776,7 +1780,7 @@ private void processUsersInRoom(List participants) { Log.d(TAG, "processUsersInRoom"); List newSessions = new ArrayList<>(); Set oldSessions = new HashSet<>(); - userIdsBySessionId = new HashMap<>(); + Map userIdsBySessionId = new HashMap<>(); hasMCU = hasExternalSignalingServer && webSocketClient != null && webSocketClient.hasMCU(); Log.d(TAG, " hasMCU is " + hasMCU); @@ -1856,15 +1860,16 @@ private void processUsersInRoom(List participants) { String userId = userIdsBySessionId.get(sessionId); if (userId != null) { - runOnUiThread(() -> { - if (participantDisplayItems.get(sessionId + "-video") != null) { - participantDisplayItems.get(sessionId + "-video").setUserId(userId); - } - if (participantDisplayItems.get(sessionId + "-screen") != null) { - participantDisplayItems.get(sessionId + "-screen").setUserId(userId); - } - }); + callParticipantModels.get(sessionId).setUserId(userId); } + + String nick; + if (hasExternalSignalingServer) { + nick = webSocketClient.getDisplayNameForSession(sessionId); + } else { + nick = offerAnswerNickProviders.get(sessionId) != null ? offerAnswerNickProviders.get(sessionId).getNick() : ""; + } + callParticipantModels.get(sessionId).setNick(nick); } if (newSessions.size() > 0 && currentCallStatus != CallStatus.IN_CONVERSATION) { @@ -1991,14 +1996,16 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp peerConnectionWrapper.addObserver(peerConnectionObserver); if (!publisher) { + MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); + if (mutableCallParticipantModel == null) { + mutableCallParticipantModel = new MutableCallParticipantModel(sessionId); + callParticipantModels.put(sessionId, mutableCallParticipantModel); + } + + final CallParticipantModel callParticipantModel = mutableCallParticipantModel; + runOnUiThread(() -> { - // userId is unknown here, but it will be got based on the session id, and the stream will be - // updated once it is added to the connection. - setupVideoStreamForLayout( - null, - sessionId, - false, - type); + setupVideoStreamForLayout(callParticipantModel, type); }); } @@ -2036,6 +2043,20 @@ private void endPeerConnection(String sessionId, boolean justScreen) { peerConnectionWrapper.removeObserver(peerConnectionObserver); runOnUiThread(() -> removeMediaStream(sessionId, videoStreamType)); + + MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); + if (mutableCallParticipantModel != null) { + if ("screen".equals(videoStreamType)) { + mutableCallParticipantModel.setScreenMediaStream(null); + mutableCallParticipantModel.setScreenIceConnectionState(null); + } else { + mutableCallParticipantModel.setMediaStream(null); + mutableCallParticipantModel.setIceConnectionState(null); + mutableCallParticipantModel.setAudioAvailable(null); + mutableCallParticipantModel.setVideoAvailable(null); + } + } + deletePeerConnection(peerConnectionWrapper); } } @@ -2051,12 +2072,19 @@ private void endPeerConnection(String sessionId, boolean justScreen) { signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); } + + callParticipantModels.remove(sessionId); } } private void removeMediaStream(String sessionId, String videoStreamType) { Log.d(TAG, "removeMediaStream"); - participantDisplayItems.remove(sessionId + "-" + videoStreamType); + ParticipantDisplayItem participantDisplayItem = participantDisplayItems.remove(sessionId + "-" + videoStreamType); + if (participantDisplayItem == null) { + return; + } + + participantDisplayItem.destroy(); if (!isDestroyed()) { initGridAdapter(); @@ -2183,40 +2211,16 @@ public void onRequestPermissionsResult(int requestCode, @NonNull String[] permis this); } - private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, - String session, - boolean videoStreamEnabled, - String videoStreamType) { - PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(session, - videoStreamType); - - PeerConnection.IceConnectionState iceConnectionState = null; - if (peerConnectionWrapper != null) { - iceConnectionState = peerConnectionWrapper.getPeerConnection().iceConnectionState(); - } - - String nick; - if (hasExternalSignalingServer) { - nick = webSocketClient.getDisplayNameForSession(session); - } else { - nick = offerAnswerNickProviders.get(session) != null ? offerAnswerNickProviders.get(session).getNick() : ""; - } - - String userId = userIdsBySessionId.get(session); - + private void setupVideoStreamForLayout(CallParticipantModel callParticipantModel, String videoStreamType) { String defaultGuestNick = getResources().getString(R.string.nc_nick_guest); ParticipantDisplayItem participantDisplayItem = new ParticipantDisplayItem(baseUrl, - userId, - session, - iceConnectionState, - nick, defaultGuestNick, - mediaStream, + rootEglBase, videoStreamType, - videoStreamEnabled, - rootEglBase); - participantDisplayItems.put(session + "-" + videoStreamType, participantDisplayItem); + callParticipantModel); + String sessionId = callParticipantModel.getSessionId(); + participantDisplayItems.put(sessionId + "-" + videoStreamType, participantDisplayItem); initGridAdapter(); } @@ -2525,11 +2529,8 @@ private OfferAnswerNickProvider(String sessionId) { private void onOfferOrAnswer(String nick) { this.nick = nick; - if (participantDisplayItems.get(sessionId + "-video") != null) { - participantDisplayItems.get(sessionId + "-video").setNick(nick); - } - if (participantDisplayItems.get(sessionId + "-screen") != null) { - participantDisplayItems.get(sessionId + "-screen").setNick(nick); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setNick(nick); } } @@ -2562,56 +2563,45 @@ public void onUnshareScreen() { private class CallActivityDataChannelMessageListener implements PeerConnectionWrapper.DataChannelMessageListener { - private final String participantDisplayItemId; + private final String sessionId; private CallActivityDataChannelMessageListener(String sessionId) { - // DataChannel messages are sent only in video peers, so the listener only acts on the "video" items. - this.participantDisplayItemId = sessionId + "-video"; + this.sessionId = sessionId; } @Override public void onAudioOn() { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(true); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setAudioAvailable(true); + } } @Override public void onAudioOff() { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(false); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setAudioAvailable(false); + } } @Override public void onVideoOn() { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(true); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setVideoAvailable(true); + } } @Override public void onVideoOff() { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(false); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setVideoAvailable(false); + } } @Override public void onNickChanged(String nick) { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setNick(nick); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setNick(nick); + } } } @@ -2619,12 +2609,10 @@ private class CallActivityPeerConnectionObserver implements PeerConnectionWrappe private final String sessionId; private final String videoStreamType; - private final String participantDisplayItemId; private CallActivityPeerConnectionObserver(String sessionId, String videoStreamType) { this.sessionId = sessionId; this.videoStreamType = videoStreamType; - this.participantDisplayItemId = sessionId + "-" + videoStreamType; } @Override @@ -2638,29 +2626,38 @@ public void onStreamRemoved(MediaStream mediaStream) { } private void handleStream(MediaStream mediaStream) { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) == null) { - return; - } + if (callParticipantModels.get(sessionId) == null) { + return; + } - boolean hasAtLeastOneVideoStream = false; - if (mediaStream != null) { - hasAtLeastOneVideoStream = mediaStream.videoTracks != null && mediaStream.videoTracks.size() > 0; - } + if ("screen".equals(videoStreamType)) { + callParticipantModels.get(sessionId).setScreenMediaStream(mediaStream); - ParticipantDisplayItem participantDisplayItem = participantDisplayItems.get(participantDisplayItemId); - participantDisplayItem.setMediaStream(mediaStream); - participantDisplayItem.setStreamEnabled(hasAtLeastOneVideoStream); - }); + return; + } + + boolean hasAtLeastOneVideoStream = false; + if (mediaStream != null) { + hasAtLeastOneVideoStream = mediaStream.videoTracks != null && mediaStream.videoTracks.size() > 0; + } + + callParticipantModels.get(sessionId).setMediaStream(mediaStream); + callParticipantModels.get(sessionId).setVideoAvailable(hasAtLeastOneVideoStream); } @Override public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { + if (callParticipantModels.get(sessionId) != null) { + if ("screen".equals(videoStreamType)) { + callParticipantModels.get(sessionId).setScreenIceConnectionState(iceConnectionState); + } else { + callParticipantModels.get(sessionId).setIceConnectionState(iceConnectionState); + } + } + runOnUiThread(() -> { if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { updateSelfVideoViewIceConnectionState(iceConnectionState); - } else if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setIceConnectionState(iceConnectionState); } if (iceConnectionState == PeerConnection.IceConnectionState.CLOSED) { diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 746f217103..6a9912eadb 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -1,7 +1,10 @@ package com.nextcloud.talk.adapters; +import android.os.Handler; +import android.os.Looper; import android.text.TextUtils; +import com.nextcloud.talk.call.CallParticipantModel; import com.nextcloud.talk.utils.ApiUtils; import org.webrtc.EglBase; @@ -14,6 +17,11 @@ public interface Observer { void onChange(); } + /** + * Shared handler to receive change notifications from the model on the main thread. + */ + private static final Handler handler = new Handler(Looper.getMainLooper()); + private final ParticipantDisplayItemNotifier participantDisplayItemNotifier = new ParticipantDisplayItemNotifier(); private final String baseUrl; @@ -23,6 +31,10 @@ public interface Observer { private final String session; private final String streamType; + private final CallParticipantModel callParticipantModel; + + private final CallParticipantModel.Observer callParticipantModelObserver = this::updateFromModel; + private String userId; private PeerConnection.IceConnectionState iceConnectionState; private String nick; @@ -31,40 +43,61 @@ public interface Observer { private boolean streamEnabled; private boolean isAudioEnabled; - public ParticipantDisplayItem(String baseUrl, String userId, String session, PeerConnection.IceConnectionState iceConnectionState, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { + public ParticipantDisplayItem(String baseUrl, String defaultGuestNick, EglBase rootEglBase, String streamType, + CallParticipantModel callParticipantModel) { this.baseUrl = baseUrl; - this.userId = userId; - this.session = session; - this.iceConnectionState = iceConnectionState; - this.nick = nick; this.defaultGuestNick = defaultGuestNick; - this.mediaStream = mediaStream; - this.streamType = streamType; - this.streamEnabled = streamEnabled; this.rootEglBase = rootEglBase; - this.updateUrlForAvatar(); + this.session = callParticipantModel.getSessionId(); + this.streamType = streamType; + + this.callParticipantModel = callParticipantModel; + this.callParticipantModel.addObserver(callParticipantModelObserver, handler); + + updateFromModel(); + } + + public void destroy() { + this.callParticipantModel.removeObserver(callParticipantModelObserver); } - public void setUserId(String userId) { - this.userId = userId; + private void updateFromModel() { + userId = callParticipantModel.getUserId(); + nick = callParticipantModel.getNick(); this.updateUrlForAvatar(); + if ("screen".equals(streamType)) { + iceConnectionState = callParticipantModel.getScreenIceConnectionState(); + mediaStream = callParticipantModel.getScreenMediaStream(); + isAudioEnabled = true; + streamEnabled = true; + } else { + iceConnectionState = callParticipantModel.getIceConnectionState(); + mediaStream = callParticipantModel.getMediaStream(); + isAudioEnabled = callParticipantModel.isAudioAvailable() != null ? + callParticipantModel.isAudioAvailable() : false; + streamEnabled = callParticipantModel.isVideoAvailable() != null ? + callParticipantModel.isVideoAvailable() : false; + } + participantDisplayItemNotifier.notifyChange(); } + private void updateUrlForAvatar() { + if (!TextUtils.isEmpty(userId)) { + urlForAvatar = ApiUtils.getUrlForAvatar(baseUrl, userId, true); + } else { + urlForAvatar = ApiUtils.getUrlForGuestAvatar(baseUrl, getNick(), true); + } + } + public boolean isConnected() { return iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; } - public void setIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { - this.iceConnectionState = iceConnectionState; - - participantDisplayItemNotifier.notifyChange(); - } - public String getNick() { if (TextUtils.isEmpty(userId) && TextUtils.isEmpty(nick)) { return defaultGuestNick; @@ -73,46 +106,18 @@ public String getNick() { return nick; } - public void setNick(String nick) { - this.nick = nick; - - this.updateUrlForAvatar(); - - participantDisplayItemNotifier.notifyChange(); - } - public String getUrlForAvatar() { return urlForAvatar; } - private void updateUrlForAvatar() { - if (!TextUtils.isEmpty(userId)) { - urlForAvatar = ApiUtils.getUrlForAvatar(baseUrl, userId, true); - } else { - urlForAvatar = ApiUtils.getUrlForGuestAvatar(baseUrl, getNick(), true); - } - } - public MediaStream getMediaStream() { return mediaStream; } - public void setMediaStream(MediaStream mediaStream) { - this.mediaStream = mediaStream; - - participantDisplayItemNotifier.notifyChange(); - } - public boolean isStreamEnabled() { return streamEnabled; } - public void setStreamEnabled(boolean streamEnabled) { - this.streamEnabled = streamEnabled; - - participantDisplayItemNotifier.notifyChange(); - } - public EglBase getRootEglBase() { return rootEglBase; } @@ -121,12 +126,6 @@ public boolean isAudioEnabled() { return isAudioEnabled; } - public void setAudioEnabled(boolean audioEnabled) { - isAudioEnabled = audioEnabled; - - participantDisplayItemNotifier.notifyChange(); - } - public void addObserver(Observer observer) { participantDisplayItemNotifier.addObserver(observer); } From 4aef76e3478c05c69edc3d25c201d1dec7f5abed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sat, 26 Nov 2022 04:03:07 +0100 Subject: [PATCH 08/19] Keep track of the stream in the peer connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../nextcloud/talk/webrtc/PeerConnectionWrapper.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index 98ef3ba5df..d040d033c1 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -121,6 +121,9 @@ public interface PeerConnectionObserver { private final boolean isMCUPublisher; private final String videoStreamType; + // It is assumed that there will be at most one remote stream at each time. + private MediaStream stream; + @Inject Context context; @@ -219,6 +222,10 @@ public String getVideoStreamType() { return videoStreamType; } + public MediaStream getStream() { + return stream; + } + public void removePeerConnection() { signalingMessageReceiver.removeListener(webRtcMessageListener); @@ -484,11 +491,15 @@ public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) { @Override public void onAddStream(MediaStream mediaStream) { + stream = mediaStream; + peerConnectionNotifier.notifyStreamAdded(mediaStream); } @Override public void onRemoveStream(MediaStream mediaStream) { + stream = null; + peerConnectionNotifier.notifyStreamRemoved(mediaStream); } From 175944e9323a3c2ac0dd4a70700539daf17fdfd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sat, 26 Nov 2022 04:18:09 +0100 Subject: [PATCH 09/19] Move handling of call participants to its own class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CallParticipant provides a read-only CallParticipantModel and internally handles the data channel and peer connection events that modify the model. Nevertheless, the CallParticipant requires certain properties to be externally set, like the userId or the peer connections. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 133 +++--------- .../nextcloud/talk/call/CallParticipant.java | 198 ++++++++++++++++++ 2 files changed, 224 insertions(+), 107 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipant.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 321b8257bb..ab5a793844 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -60,8 +60,8 @@ import com.nextcloud.talk.adapters.ParticipantsAdapter; import com.nextcloud.talk.api.NcApi; import com.nextcloud.talk.application.NextcloudTalkApplication; +import com.nextcloud.talk.call.CallParticipant; import com.nextcloud.talk.call.CallParticipantModel; -import com.nextcloud.talk.call.MutableCallParticipantModel; import com.nextcloud.talk.data.user.model.User; import com.nextcloud.talk.databinding.CallActivityBinding; import com.nextcloud.talk.events.ConfigurationChangeEvent; @@ -264,11 +264,9 @@ public class CallActivity extends CallBaseActivity { private Map callParticipantMessageListeners = new HashMap<>(); - private Map dataChannelMessageListeners = new HashMap<>(); - private Map peerConnectionObservers = new HashMap<>(); - private Map callParticipantModels = new HashMap<>(); + private Map callParticipants = new HashMap<>(); private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { @@ -385,7 +383,7 @@ public void onCreate(Bundle savedInstanceState) { requestBluetoothPermission(); } basicInitialization(); - callParticipantModels = new HashMap<>(); + callParticipants = new HashMap<>(); participantDisplayItems = new HashMap<>(); initViews(); if (!isConnectionEstablished()) { @@ -1860,7 +1858,7 @@ private void processUsersInRoom(List participants) { String userId = userIdsBySessionId.get(sessionId); if (userId != null) { - callParticipantModels.get(sessionId).setUserId(userId); + callParticipants.get(sessionId).setUserId(userId); } String nick; @@ -1869,7 +1867,7 @@ private void processUsersInRoom(List participants) { } else { nick = offerAnswerNickProviders.get(sessionId) != null ? offerAnswerNickProviders.get(sessionId).getNick() : ""; } - callParticipantModels.get(sessionId).setNick(nick); + callParticipants.get(sessionId).setNick(nick); } if (newSessions.size() > 0 && currentCallStatus != CallStatus.IN_CONVERSATION) { @@ -1975,12 +1973,6 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp new CallActivityCallParticipantMessageListener(sessionId); callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); - - // DataChannel messages are sent only in video peers; (sender) screen peers do not even open them. - PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = - new CallActivityDataChannelMessageListener(sessionId); - dataChannelMessageListeners.put(sessionId, dataChannelMessageListener); - peerConnectionWrapper.addListener(dataChannelMessageListener); } if (!publisher && !hasExternalSignalingServer && offerAnswerNickProviders.get(sessionId) == null) { @@ -1996,13 +1988,19 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp peerConnectionWrapper.addObserver(peerConnectionObserver); if (!publisher) { - MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); - if (mutableCallParticipantModel == null) { - mutableCallParticipantModel = new MutableCallParticipantModel(sessionId); - callParticipantModels.put(sessionId, mutableCallParticipantModel); + CallParticipant callParticipant = callParticipants.get(sessionId); + if (callParticipant == null) { + callParticipant = new CallParticipant(sessionId); + callParticipants.put(sessionId, callParticipant); } - final CallParticipantModel callParticipantModel = mutableCallParticipantModel; + if ("screen".equals(type)) { + callParticipant.setScreenPeerConnectionWrapper(peerConnectionWrapper); + } else { + callParticipant.setPeerConnectionWrapper(peerConnectionWrapper); + } + + final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); runOnUiThread(() -> { setupVideoStreamForLayout(callParticipantModel, type); @@ -2033,10 +2031,6 @@ private void endPeerConnection(String sessionId, boolean justScreen) { if (!(peerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty()) { for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrappers) { if (peerConnectionWrapper.getSessionId().equals(sessionId)) { - if (!justScreen && VIDEO_STREAM_TYPE_VIDEO.equals(peerConnectionWrapper.getVideoStreamType())) { - PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = dataChannelMessageListeners.remove(sessionId); - peerConnectionWrapper.removeListener(dataChannelMessageListener); - } String videoStreamType = peerConnectionWrapper.getVideoStreamType(); if (VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType) || !justScreen) { PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = peerConnectionObservers.remove(sessionId + "-" + videoStreamType); @@ -2044,16 +2038,12 @@ private void endPeerConnection(String sessionId, boolean justScreen) { runOnUiThread(() -> removeMediaStream(sessionId, videoStreamType)); - MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); - if (mutableCallParticipantModel != null) { + CallParticipant callParticipant = callParticipants.get(sessionId); + if (callParticipant != null) { if ("screen".equals(videoStreamType)) { - mutableCallParticipantModel.setScreenMediaStream(null); - mutableCallParticipantModel.setScreenIceConnectionState(null); + callParticipant.setScreenPeerConnectionWrapper(null); } else { - mutableCallParticipantModel.setMediaStream(null); - mutableCallParticipantModel.setIceConnectionState(null); - mutableCallParticipantModel.setAudioAvailable(null); - mutableCallParticipantModel.setVideoAvailable(null); + callParticipant.setPeerConnectionWrapper(null); } } @@ -2073,7 +2063,10 @@ private void endPeerConnection(String sessionId, boolean justScreen) { signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); } - callParticipantModels.remove(sessionId); + CallParticipant callParticipant = callParticipants.remove(sessionId); + if (callParticipant != null) { + callParticipant.destroy(); + } } } @@ -2529,8 +2522,8 @@ private OfferAnswerNickProvider(String sessionId) { private void onOfferOrAnswer(String nick) { this.nick = nick; - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setNick(nick); + if (callParticipants.get(sessionId) != null) { + callParticipants.get(sessionId).setNick(nick); } } @@ -2561,50 +2554,6 @@ public void onUnshareScreen() { } } - private class CallActivityDataChannelMessageListener implements PeerConnectionWrapper.DataChannelMessageListener { - - private final String sessionId; - - private CallActivityDataChannelMessageListener(String sessionId) { - this.sessionId = sessionId; - } - - @Override - public void onAudioOn() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setAudioAvailable(true); - } - } - - @Override - public void onAudioOff() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setAudioAvailable(false); - } - } - - @Override - public void onVideoOn() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setVideoAvailable(true); - } - } - - @Override - public void onVideoOff() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setVideoAvailable(false); - } - } - - @Override - public void onNickChanged(String nick) { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setNick(nick); - } - } - } - private class CallActivityPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver { private final String sessionId; @@ -2617,44 +2566,14 @@ private CallActivityPeerConnectionObserver(String sessionId, String videoStreamT @Override public void onStreamAdded(MediaStream mediaStream) { - handleStream(mediaStream); } @Override public void onStreamRemoved(MediaStream mediaStream) { - handleStream(null); - } - - private void handleStream(MediaStream mediaStream) { - if (callParticipantModels.get(sessionId) == null) { - return; - } - - if ("screen".equals(videoStreamType)) { - callParticipantModels.get(sessionId).setScreenMediaStream(mediaStream); - - return; - } - - boolean hasAtLeastOneVideoStream = false; - if (mediaStream != null) { - hasAtLeastOneVideoStream = mediaStream.videoTracks != null && mediaStream.videoTracks.size() > 0; - } - - callParticipantModels.get(sessionId).setMediaStream(mediaStream); - callParticipantModels.get(sessionId).setVideoAvailable(hasAtLeastOneVideoStream); } @Override public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { - if (callParticipantModels.get(sessionId) != null) { - if ("screen".equals(videoStreamType)) { - callParticipantModels.get(sessionId).setScreenIceConnectionState(iceConnectionState); - } else { - callParticipantModels.get(sessionId).setIceConnectionState(iceConnectionState); - } - } - runOnUiThread(() -> { if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { updateSelfVideoViewIceConnectionState(iceConnectionState); diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java new file mode 100644 index 0000000000..3b153f8cbf --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java @@ -0,0 +1,198 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.webrtc.PeerConnectionWrapper; + +import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; + +/** + * Model for (remote) call participants. + * + * This class keeps track of the state changes in a call participant and updates its data model as needed. View classes + * are expected to directly use the read-only data model. + */ +public class CallParticipant { + + private final PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = + new PeerConnectionWrapper.PeerConnectionObserver() { + @Override + public void onStreamAdded(MediaStream mediaStream) { + handleStreamChange(mediaStream); + } + + @Override + public void onStreamRemoved(MediaStream mediaStream) { + handleStreamChange(mediaStream); + } + + @Override + public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { + handleIceConnectionStateChange(iceConnectionState); + } + }; + + private final PeerConnectionWrapper.PeerConnectionObserver screenPeerConnectionObserver = + new PeerConnectionWrapper.PeerConnectionObserver() { + @Override + public void onStreamAdded(MediaStream mediaStream) { + callParticipantModel.setScreenMediaStream(mediaStream); + } + + @Override + public void onStreamRemoved(MediaStream mediaStream) { + callParticipantModel.setScreenMediaStream(null); + } + + @Override + public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { + callParticipantModel.setScreenIceConnectionState(iceConnectionState); + } + }; + + // DataChannel messages are sent only in video peers; (sender) screen peers do not even open them. + private final PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = + new PeerConnectionWrapper.DataChannelMessageListener() { + @Override + public void onAudioOn() { + callParticipantModel.setAudioAvailable(Boolean.TRUE); + } + + @Override + public void onAudioOff() { + callParticipantModel.setAudioAvailable(Boolean.FALSE); + } + + @Override + public void onVideoOn() { + callParticipantModel.setVideoAvailable(Boolean.TRUE); + } + + @Override + public void onVideoOff() { + callParticipantModel.setVideoAvailable(Boolean.FALSE); + } + + @Override + public void onNickChanged(String nick) { + callParticipantModel.setNick(nick); + } + }; + + private final MutableCallParticipantModel callParticipantModel; + + private PeerConnectionWrapper peerConnectionWrapper; + private PeerConnectionWrapper screenPeerConnectionWrapper; + + public CallParticipant(String sessionId) { + callParticipantModel = new MutableCallParticipantModel(sessionId); + } + + public void destroy() { + if (peerConnectionWrapper != null) { + peerConnectionWrapper.removeObserver(peerConnectionObserver); + peerConnectionWrapper.removeListener(dataChannelMessageListener); + } + if (screenPeerConnectionWrapper != null) { + screenPeerConnectionWrapper.removeObserver(screenPeerConnectionObserver); + } + } + + public CallParticipantModel getCallParticipantModel() { + return callParticipantModel; + } + + public void setUserId(String userId) { + callParticipantModel.setUserId(userId); + } + + public void setNick(String nick) { + callParticipantModel.setNick(nick); + } + + public void setPeerConnectionWrapper(PeerConnectionWrapper peerConnectionWrapper) { + if (this.peerConnectionWrapper != null) { + this.peerConnectionWrapper.removeObserver(peerConnectionObserver); + this.peerConnectionWrapper.removeListener(dataChannelMessageListener); + } + + this.peerConnectionWrapper = peerConnectionWrapper; + + if (this.peerConnectionWrapper == null) { + callParticipantModel.setIceConnectionState(null); + callParticipantModel.setMediaStream(null); + callParticipantModel.setAudioAvailable(null); + callParticipantModel.setVideoAvailable(null); + + return; + } + + handleIceConnectionStateChange(this.peerConnectionWrapper.getPeerConnection().iceConnectionState()); + handleStreamChange(this.peerConnectionWrapper.getStream()); + + this.peerConnectionWrapper.addObserver(peerConnectionObserver); + this.peerConnectionWrapper.addListener(dataChannelMessageListener); + } + + private void handleIceConnectionStateChange(PeerConnection.IceConnectionState iceConnectionState) { + callParticipantModel.setIceConnectionState(iceConnectionState); + + if (iceConnectionState == PeerConnection.IceConnectionState.NEW || + iceConnectionState == PeerConnection.IceConnectionState.CHECKING) { + callParticipantModel.setAudioAvailable(null); + callParticipantModel.setVideoAvailable(null); + } + } + + private void handleStreamChange(MediaStream mediaStream) { + if (mediaStream == null) { + callParticipantModel.setMediaStream(null); + callParticipantModel.setVideoAvailable(Boolean.FALSE); + + return; + } + + boolean hasAtLeastOneVideoStream = mediaStream.videoTracks != null && !mediaStream.videoTracks.isEmpty(); + + callParticipantModel.setMediaStream(mediaStream); + callParticipantModel.setVideoAvailable(hasAtLeastOneVideoStream); + } + + public void setScreenPeerConnectionWrapper(PeerConnectionWrapper screenPeerConnectionWrapper) { + if (this.screenPeerConnectionWrapper != null) { + this.screenPeerConnectionWrapper.removeObserver(screenPeerConnectionObserver); + } + + this.screenPeerConnectionWrapper = screenPeerConnectionWrapper; + + if (this.screenPeerConnectionWrapper == null) { + callParticipantModel.setScreenIceConnectionState(null); + callParticipantModel.setScreenMediaStream(null); + + return; + } + + callParticipantModel.setScreenIceConnectionState(this.screenPeerConnectionWrapper.getPeerConnection().iceConnectionState()); + callParticipantModel.setScreenMediaStream(this.screenPeerConnectionWrapper.getStream()); + + this.screenPeerConnectionWrapper.addObserver(screenPeerConnectionObserver); + } +} From 5681084a14f0b59b82da217a367b546f9bcb6d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 19:19:44 +0100 Subject: [PATCH 10/19] Create and destroy helper listeners based on call participants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The listeners for call participant messages and for the call participant nick provided by offers / answers were created and destroyed based on the peer connections, although they were implicitly associated to a call participant. Now they are explicitly created and destroyed based on its associated call participant. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 46 +++++++++---------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index ab5a793844..93152fdf3c 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1966,22 +1966,6 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp peerConnectionWrapperList.add(peerConnectionWrapper); - // Currently there is no separation between call participants and peer connections, so any video peer - // connection (except the own publisher connection) is treated as a call participant. - if (!publisher && "video".equals(type)) { - SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener = - new CallActivityCallParticipantMessageListener(sessionId); - callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); - signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); - } - - if (!publisher && !hasExternalSignalingServer && offerAnswerNickProviders.get(sessionId) == null) { - OfferAnswerNickProvider offerAnswerNickProvider = new OfferAnswerNickProvider(sessionId); - offerAnswerNickProviders.put(sessionId, offerAnswerNickProvider); - signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); - signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); - } - PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = new CallActivityPeerConnectionObserver(sessionId, type); peerConnectionObservers.put(sessionId + "-" + type, peerConnectionObserver); @@ -1992,6 +1976,18 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp if (callParticipant == null) { callParticipant = new CallParticipant(sessionId); callParticipants.put(sessionId, callParticipant); + + SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener = + new CallActivityCallParticipantMessageListener(sessionId); + callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); + signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); + + if (!hasExternalSignalingServer) { + OfferAnswerNickProvider offerAnswerNickProvider = new OfferAnswerNickProvider(sessionId); + offerAnswerNickProviders.put(sessionId, offerAnswerNickProvider); + signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); + signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); + } } if ("screen".equals(type)) { @@ -2054,18 +2050,18 @@ private void endPeerConnection(String sessionId, boolean justScreen) { } if (!justScreen) { - SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); - signalingMessageReceiver.removeListener(listener); - - OfferAnswerNickProvider offerAnswerNickProvider = offerAnswerNickProviders.remove(sessionId); - if (offerAnswerNickProvider != null) { - signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); - signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); - } - CallParticipant callParticipant = callParticipants.remove(sessionId); if (callParticipant != null) { callParticipant.destroy(); + + SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); + signalingMessageReceiver.removeListener(listener); + + OfferAnswerNickProvider offerAnswerNickProvider = offerAnswerNickProviders.remove(sessionId); + if (offerAnswerNickProvider != null) { + signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); + signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); + } } } } From e17a999812e91ce74bb1c6c77ec345d2279ba312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sun, 27 Nov 2022 21:58:49 +0100 Subject: [PATCH 11/19] Rename methods to add and remove ParticipantDisplayItems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../com/nextcloud/talk/activities/CallActivity.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 93152fdf3c..029aaf29dc 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1999,7 +1999,7 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); runOnUiThread(() -> { - setupVideoStreamForLayout(callParticipantModel, type); + addParticipantDisplayItem(callParticipantModel, type); }); } @@ -2032,7 +2032,7 @@ private void endPeerConnection(String sessionId, boolean justScreen) { PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = peerConnectionObservers.remove(sessionId + "-" + videoStreamType); peerConnectionWrapper.removeObserver(peerConnectionObserver); - runOnUiThread(() -> removeMediaStream(sessionId, videoStreamType)); + runOnUiThread(() -> removeParticipantDisplayItem(sessionId, videoStreamType)); CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant != null) { @@ -2066,8 +2066,8 @@ private void endPeerConnection(String sessionId, boolean justScreen) { } } - private void removeMediaStream(String sessionId, String videoStreamType) { - Log.d(TAG, "removeMediaStream"); + private void removeParticipantDisplayItem(String sessionId, String videoStreamType) { + Log.d(TAG, "removeParticipantDisplayItem"); ParticipantDisplayItem participantDisplayItem = participantDisplayItems.remove(sessionId + "-" + videoStreamType); if (participantDisplayItem == null) { return; @@ -2200,7 +2200,7 @@ public void onRequestPermissionsResult(int requestCode, @NonNull String[] permis this); } - private void setupVideoStreamForLayout(CallParticipantModel callParticipantModel, String videoStreamType) { + private void addParticipantDisplayItem(CallParticipantModel callParticipantModel, String videoStreamType) { String defaultGuestNick = getResources().getString(R.string.nc_nick_guest); ParticipantDisplayItem participantDisplayItem = new ParticipantDisplayItem(baseUrl, From 534bbddc88fa986406399c9e9e31975264d93bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sun, 27 Nov 2022 22:12:46 +0100 Subject: [PATCH 12/19] Create and destroy ParticipantDisplayItems based on call participants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ParticipantDisplayItems were created and destroyed based on the peer connections. Now a ParticipantDisplayItem of "video" type is associated to a call participant, while an additional item is created and destroyed depending on the state of the screen peer connection of the call participant. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 53 ++++++++++++++++--- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 029aaf29dc..343111e8d3 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -268,6 +268,10 @@ public class CallActivity extends CallBaseActivity { private Map callParticipants = new HashMap<>(); + private Map screenParticipantDisplayItemManagers = new HashMap<>(); + + private Handler screenParticipantDisplayItemManagersHandler = new Handler(Looper.getMainLooper()); + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { @Override @@ -1988,6 +1992,17 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); } + + final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); + + ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = + new ScreenParticipantDisplayItemManager(callParticipantModel); + screenParticipantDisplayItemManagers.put(sessionId, screenParticipantDisplayItemManager); + callParticipantModel.addObserver(screenParticipantDisplayItemManager, screenParticipantDisplayItemManagersHandler); + + runOnUiThread(() -> { + addParticipantDisplayItem(callParticipantModel, "video"); + }); } if ("screen".equals(type)) { @@ -1995,12 +2010,6 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp } else { callParticipant.setPeerConnectionWrapper(peerConnectionWrapper); } - - final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); - - runOnUiThread(() -> { - addParticipantDisplayItem(callParticipantModel, type); - }); } if (publisher) { @@ -2032,8 +2041,6 @@ private void endPeerConnection(String sessionId, boolean justScreen) { PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = peerConnectionObservers.remove(sessionId + "-" + videoStreamType); peerConnectionWrapper.removeObserver(peerConnectionObserver); - runOnUiThread(() -> removeParticipantDisplayItem(sessionId, videoStreamType)); - CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant != null) { if ("screen".equals(videoStreamType)) { @@ -2052,6 +2059,10 @@ private void endPeerConnection(String sessionId, boolean justScreen) { if (!justScreen) { CallParticipant callParticipant = callParticipants.remove(sessionId); if (callParticipant != null) { + ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = + screenParticipantDisplayItemManagers.remove(sessionId); + callParticipant.getCallParticipantModel().removeObserver(screenParticipantDisplayItemManager); + callParticipant.destroy(); SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); @@ -2062,6 +2073,8 @@ private void endPeerConnection(String sessionId, boolean justScreen) { signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); } + + runOnUiThread(() -> removeParticipantDisplayItem(sessionId, "video")); } } } @@ -2594,6 +2607,30 @@ public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceCon } } + private class ScreenParticipantDisplayItemManager implements CallParticipantModel.Observer { + + private final CallParticipantModel callParticipantModel; + + private ScreenParticipantDisplayItemManager(CallParticipantModel callParticipantModel) { + this.callParticipantModel = callParticipantModel; + } + + @Override + public void onChange() { + String sessionId = callParticipantModel.getSessionId(); + if (callParticipantModel.getScreenIceConnectionState() == null) { + removeParticipantDisplayItem(sessionId, "screen"); + + return; + } + + boolean hasScreenParticipantDisplayItem = participantDisplayItems.get(sessionId + "-screen") != null; + if (!hasScreenParticipantDisplayItem) { + addParticipantDisplayItem(callParticipantModel, "screen"); + } + } + } + private class InternalSignalingMessageSender implements SignalingMessageSender { @Override From 6728e3f063b18cdc173f930267dbc83e4191f85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 28 Nov 2022 09:12:27 +0100 Subject: [PATCH 13/19] Do not handle connection state changes to "closed" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The connection state changes to "closed" only when the connection is closed. However, closing a connection does not fire any event (not even the "iceConnectionStateChanged" event), so the event handler can be removed as it will never be executed. Signed-off-by: Daniel Calviño Sánchez --- .../com/nextcloud/talk/activities/CallActivity.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 343111e8d3..f342be713b 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1971,7 +1971,7 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp peerConnectionWrapperList.add(peerConnectionWrapper); PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = - new CallActivityPeerConnectionObserver(sessionId, type); + new CallActivityPeerConnectionObserver(sessionId); peerConnectionObservers.put(sessionId + "-" + type, peerConnectionObserver); peerConnectionWrapper.addObserver(peerConnectionObserver); @@ -2566,11 +2566,9 @@ public void onUnshareScreen() { private class CallActivityPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver { private final String sessionId; - private final String videoStreamType; - private CallActivityPeerConnectionObserver(String sessionId, String videoStreamType) { + private CallActivityPeerConnectionObserver(String sessionId) { this.sessionId = sessionId; - this.videoStreamType = videoStreamType; } @Override @@ -2588,12 +2586,6 @@ public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceCon updateSelfVideoViewIceConnectionState(iceConnectionState); } - if (iceConnectionState == PeerConnection.IceConnectionState.CLOSED) { - endPeerConnection(sessionId, VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType)); - - return; - } - if (iceConnectionState == PeerConnection.IceConnectionState.FAILED) { if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { setCallState(CallStatus.PUBLISHER_FAILED); From 0a3f515bb6e1bfd5475177f716b53590577e46db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 28 Nov 2022 10:01:33 +0100 Subject: [PATCH 14/19] Observe only the self peer connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The observers were created for any peer connection, but after recent changes they ignored all changes but those from the self peer connection. Therefore it is enough to just add an explicit listener on that peer connection rather than on all of them. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index f342be713b..df89f8b0b2 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -264,7 +264,7 @@ public class CallActivity extends CallBaseActivity { private Map callParticipantMessageListeners = new HashMap<>(); - private Map peerConnectionObservers = new HashMap<>(); + private PeerConnectionWrapper.PeerConnectionObserver selfPeerConnectionObserver = new CallActivitySelfPeerConnectionObserver(); private Map callParticipants = new HashMap<>(); @@ -1970,11 +1970,6 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp peerConnectionWrapperList.add(peerConnectionWrapper); - PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = - new CallActivityPeerConnectionObserver(sessionId); - peerConnectionObservers.put(sessionId + "-" + type, peerConnectionObserver); - peerConnectionWrapper.addObserver(peerConnectionObserver); - if (!publisher) { CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant == null) { @@ -2013,6 +2008,8 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp } if (publisher) { + peerConnectionWrapper.addObserver(selfPeerConnectionObserver); + startSendingNick(); } @@ -2036,11 +2033,12 @@ private void endPeerConnection(String sessionId, boolean justScreen) { if (!(peerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty()) { for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrappers) { if (peerConnectionWrapper.getSessionId().equals(sessionId)) { + if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { + peerConnectionWrapper.removeObserver(selfPeerConnectionObserver); + } + String videoStreamType = peerConnectionWrapper.getVideoStreamType(); if (VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType) || !justScreen) { - PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = peerConnectionObservers.remove(sessionId + "-" + videoStreamType); - peerConnectionWrapper.removeObserver(peerConnectionObserver); - CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant != null) { if ("screen".equals(videoStreamType)) { @@ -2563,13 +2561,7 @@ public void onUnshareScreen() { } } - private class CallActivityPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver { - - private final String sessionId; - - private CallActivityPeerConnectionObserver(String sessionId) { - this.sessionId = sessionId; - } + private class CallActivitySelfPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver { @Override public void onStreamAdded(MediaStream mediaStream) { @@ -2582,18 +2574,12 @@ public void onStreamRemoved(MediaStream mediaStream) { @Override public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { runOnUiThread(() -> { - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - updateSelfVideoViewIceConnectionState(iceConnectionState); - } + updateSelfVideoViewIceConnectionState(iceConnectionState); if (iceConnectionState == PeerConnection.IceConnectionState.FAILED) { - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - setCallState(CallStatus.PUBLISHER_FAILED); - webSocketClient.clearResumeId(); - hangup(false); - } - - return; + setCallState(CallStatus.PUBLISHER_FAILED); + webSocketClient.clearResumeId(); + hangup(false); } }); } From ab72db7a10538f12c0213904f1527f8c5058a71c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 13:04:17 +0100 Subject: [PATCH 15/19] Add helper class to keep track of the participants in a call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For now only the same signaling messages that were already handled are still handled; in the future it could be extended to handle other messages, like the one sent by the external signaling server when a participant leaves the room (in some cases no participants update message is sent if the participant leaves the call and room at the same time, which causes the participants to still be seen as in call until a new update is received). Signed-off-by: Daniel Calviño Sánchez --- .../talk/call/CallParticipantList.java | 164 +++++ .../call/CallParticipantListNotifier.java | 63 ++ ...lParticipantListExternalSignalingTest.java | 663 ++++++++++++++++++ ...lParticipantListInternalSignalingTest.java | 535 ++++++++++++++ .../talk/call/CallParticipantListTest.java | 61 ++ 5 files changed, 1486 insertions(+) create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipantList.java create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipantListNotifier.java create mode 100644 app/src/test/java/com/nextcloud/talk/call/CallParticipantListExternalSignalingTest.java create mode 100644 app/src/test/java/com/nextcloud/talk/call/CallParticipantListInternalSignalingTest.java create mode 100644 app/src/test/java/com/nextcloud/talk/call/CallParticipantListTest.java diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantList.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantList.java new file mode 100644 index 0000000000..6135ab9910 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantList.java @@ -0,0 +1,164 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.models.json.participants.Participant; +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Helper class to keep track of the participants in a call based on the signaling messages. + * + * The CallParticipantList adds a listener for participant list messages as soon as it is created and starts tracking + * the call participants until destroyed. Notifications about the changes can be received by adding an observer to the + * CallParticipantList; note that no sorting is guaranteed on the participants. + */ +public class CallParticipantList { + + public interface Observer { + void onCallParticipantsChanged(Collection joined, Collection updated, + Collection left, Collection unchanged); + void onCallEndedForAll(); + } + + private final SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = + new SignalingMessageReceiver.ParticipantListMessageListener() { + + private final Map callParticipants = new HashMap<>(); + + @Override + public void onUsersInRoom(List participants) { + processParticipantList(participants); + } + + @Override + public void onParticipantsUpdate(List participants) { + processParticipantList(participants); + } + + private void processParticipantList(List participants) { + Collection joined = new ArrayList<>(); + Collection updated = new ArrayList<>(); + Collection left = new ArrayList<>(); + Collection unchanged = new ArrayList<>(); + + Collection knownCallParticipantsNotFound = new ArrayList<>(callParticipants.values()); + + for (Participant participant : participants) { + String sessionId = participant.getSessionId(); + Participant callParticipant = callParticipants.get(sessionId); + + boolean knownCallParticipant = callParticipant != null; + if (!knownCallParticipant && participant.getInCall() != Participant.InCallFlags.DISCONNECTED) { + callParticipants.put(sessionId, copyParticipant(participant)); + joined.add(copyParticipant(participant)); + } else if (knownCallParticipant && participant.getInCall() == Participant.InCallFlags.DISCONNECTED) { + callParticipants.remove(sessionId); + // No need to copy it, as it will be no longer used. + callParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + left.add(callParticipant); + } else if (knownCallParticipant && callParticipant.getInCall() != participant.getInCall()) { + callParticipant.setInCall(participant.getInCall()); + updated.add(copyParticipant(participant)); + } else if (knownCallParticipant) { + unchanged.add(copyParticipant(participant)); + } + + if (knownCallParticipant) { + knownCallParticipantsNotFound.remove(callParticipant); + } + } + + for (Participant callParticipant : knownCallParticipantsNotFound) { + callParticipants.remove(callParticipant.getSessionId()); + // No need to copy it, as it will be no longer used. + callParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + left.add(callParticipant); + } + + if (!joined.isEmpty() || !updated.isEmpty() || !left.isEmpty()) { + callParticipantListNotifier.notifyChanged(joined, updated, left, unchanged); + } + } + + @Override + public void onAllParticipantsUpdate(long inCall) { + if (inCall != Participant.InCallFlags.DISCONNECTED) { + // Updating all participants is expected to happen only to disconnect them. + return; + } + + callParticipantListNotifier.notifyCallEndedForAll(); + + Collection joined = new ArrayList<>(); + Collection updated = new ArrayList<>(); + Collection left = new ArrayList<>(callParticipants.size()); + Collection unchanged = new ArrayList<>(); + + for (Participant callParticipant : callParticipants.values()) { + // No need to copy it, as it will be no longer used. + callParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + left.add(callParticipant); + } + callParticipants.clear(); + + if (!left.isEmpty()) { + callParticipantListNotifier.notifyChanged(joined, updated, left, unchanged); + } + } + + private Participant copyParticipant(Participant participant) { + Participant copiedParticipant = new Participant(); + copiedParticipant.setInCall(participant.getInCall()); + copiedParticipant.setLastPing(participant.getLastPing()); + copiedParticipant.setSessionId(participant.getSessionId()); + copiedParticipant.setType(participant.getType()); + copiedParticipant.setUserId(participant.getUserId()); + + return copiedParticipant; + } + }; + + private final CallParticipantListNotifier callParticipantListNotifier = new CallParticipantListNotifier(); + + private final SignalingMessageReceiver signalingMessageReceiver; + + public CallParticipantList(SignalingMessageReceiver signalingMessageReceiver) { + this.signalingMessageReceiver = signalingMessageReceiver; + this.signalingMessageReceiver.addListener(participantListMessageListener); + } + + public void destroy() { + signalingMessageReceiver.removeListener(participantListMessageListener); + } + + public void addObserver(Observer observer) { + callParticipantListNotifier.addObserver(observer); + } + + public void removeObserver(Observer observer) { + callParticipantListNotifier.removeObserver(observer); + } +} diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantListNotifier.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantListNotifier.java new file mode 100644 index 0000000000..afbc893fd7 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantListNotifier.java @@ -0,0 +1,63 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.models.json.participants.Participant; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Helper class to register and notify CallParticipantList.Observers. + * + * This class is only meant for internal use by CallParticipantList; listeners must register themselves against + * a CallParticipantList rather than against a CallParticipantListNotifier. + */ +class CallParticipantListNotifier { + + private final Set callParticipantListObservers = new LinkedHashSet<>(); + + public synchronized void addObserver(CallParticipantList.Observer observer) { + if (observer == null) { + throw new IllegalArgumentException("CallParticipantList.Observer can not be null"); + } + + callParticipantListObservers.add(observer); + } + + public synchronized void removeObserver(CallParticipantList.Observer observer) { + callParticipantListObservers.remove(observer); + } + + public synchronized void notifyChanged(Collection joined, Collection updated, + Collection left, Collection unchanged) { + for (CallParticipantList.Observer observer : new ArrayList<>(callParticipantListObservers)) { + observer.onCallParticipantsChanged(joined, updated, left, unchanged); + } + } + + public synchronized void notifyCallEndedForAll() { + for (CallParticipantList.Observer observer : new ArrayList<>(callParticipantListObservers)) { + observer.onCallEndedForAll(); + } + } +} diff --git a/app/src/test/java/com/nextcloud/talk/call/CallParticipantListExternalSignalingTest.java b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListExternalSignalingTest.java new file mode 100644 index 0000000000..7d20122078 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListExternalSignalingTest.java @@ -0,0 +1,663 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.models.json.participants.Participant; +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatcher; +import org.mockito.InOrder; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.DISCONNECTED; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.IN_CALL; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.WITH_AUDIO; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.WITH_VIDEO; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.GUEST; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.GUEST_MODERATOR; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.MODERATOR; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.OWNER; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.USER; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.only; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +public class CallParticipantListExternalSignalingTest { + + private static class ParticipantsUpdateParticipantBuilder { + private Participant newUser(long inCall, long lastPing, String sessionId, Participant.ParticipantType type, + String userId) { + Participant participant = new Participant(); + participant.setInCall(inCall); + participant.setLastPing(lastPing); + participant.setSessionId(sessionId); + participant.setType(type); + participant.setUserId(userId); + + return participant; + } + + private Participant newGuest(long inCall, long lastPing, String sessionId, Participant.ParticipantType type) { + Participant participant = new Participant(); + participant.setInCall(inCall); + participant.setLastPing(lastPing); + participant.setSessionId(sessionId); + participant.setType(type); + + return participant; + } + } + + private final ParticipantsUpdateParticipantBuilder builder = new ParticipantsUpdateParticipantBuilder(); + + private CallParticipantList callParticipantList; + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener; + + private CallParticipantList.Observer mockedCallParticipantListObserver; + + private Collection expectedJoined; + private Collection expectedUpdated; + private Collection expectedLeft; + private Collection expectedUnchanged; + + // The order of the left participants in some tests depends on how they are internally sorted by the map, so the + // list of left participants needs to be checked ignoring the sorting (or, rather, sorting by session ID as in + // expectedLeft). + // Other tests can just relay on the not guaranteed, but known internal sorting of the elements. + private final ArgumentMatcher> matchesExpectedLeftIgnoringOrder = left -> { + Collections.sort(left, Comparator.comparing(Participant::getSessionId)); + return expectedLeft.equals(left); + }; + + @Before + public void setUp() { + SignalingMessageReceiver mockedSignalingMessageReceiver = mock(SignalingMessageReceiver.class); + + callParticipantList = new CallParticipantList(mockedSignalingMessageReceiver); + + mockedCallParticipantListObserver = mock(CallParticipantList.Observer.class); + + // Get internal ParticipantListMessageListener from callParticipantList set in the + // mockedSignalingMessageReceiver. + ArgumentCaptor participantListMessageListenerArgumentCaptor = + ArgumentCaptor.forClass(SignalingMessageReceiver.ParticipantListMessageListener.class); + + verify(mockedSignalingMessageReceiver).addListener(participantListMessageListenerArgumentCaptor.capture()); + + participantListMessageListener = participantListMessageListenerArgumentCaptor.getValue(); + + expectedJoined = new ArrayList<>(); + expectedUpdated = new ArrayList<>(); + expectedLeft = new ArrayList<>(); + expectedUnchanged = new ArrayList<>(); + } + + @Test + public void testParticipantsUpdateJoinRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateJoinRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(DISCONNECTED, 4, "theSessionId4", USER, "theUserId4")); + participants.add(builder.newUser(DISCONNECTED, 5, "theSessionId5", OWNER, "theUserId5")); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateJoinRoomThenJoinCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateJoinRoomThenJoinCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(DISCONNECTED, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedJoined.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateJoinRoomAndCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateJoinRoomAndCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedJoined.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateJoinRoomAndCallRepeated() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + participantListMessageListener.onParticipantsUpdate(participants); + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateChangeCallFlags() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedUpdated.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateChangeCallFlagsSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL | WITH_VIDEO, 4, "theSessionId4", USER, "theUserId4")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedUpdated.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedUpdated.add(builder.newUser(IN_CALL | WITH_VIDEO, 4, "theSessionId4", USER, "theUserId4")); + expectedUnchanged.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2", GUEST)); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateChangeLastPing() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateChangeLastPingSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", USER, "theUserId3")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 108, "theSessionId2", GUEST)); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 815, "theSessionId3", USER, "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateChangeParticipantType() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", USER, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateChangeParticipantTypeeSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", USER, "theUserId3")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", USER, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 2, "theSessionId2", GUEST_MODERATOR)); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", MODERATOR, "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateLeaveCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateLeaveCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateLeaveCallThenLeaveRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateLeaveCallThenLeaveRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateLeaveCallAndRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateLeaveCallAndRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } + + @Test + public void testParticipantsUpdateSeveralEventsSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + participants.add(builder.newUser(IN_CALL, 5, "theSessionId5", OWNER, "theUserId5")); + // theSessionId6 has not joined yet. + participants.add(builder.newGuest(IN_CALL | WITH_VIDEO, 7, "theSessionId7", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 8, "theSessionId8", USER, "theUserId8")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 9, "theSessionId9", MODERATOR, "theUserId9")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + // theSessionId1 is gone. + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 5, "theSessionId5", OWNER, "theUserId5")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 6, "theSessionId6", GUEST)); + participants.add(builder.newGuest(IN_CALL, 7, "theSessionId7", GUEST)); + participants.add(builder.newUser(IN_CALL, 8, "theSessionId8", USER, "theUserId8")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId9", USER, "theUserId9")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newGuest(IN_CALL | WITH_AUDIO, 6, "theSessionId6", GUEST)); + expectedJoined.add(builder.newUser(IN_CALL, 8, "theSessionId8", USER, "theUserId8")); + expectedUpdated.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 5, "theSessionId5", OWNER, "theUserId5")); + expectedUpdated.add(builder.newGuest(IN_CALL, 7, "theSessionId7", GUEST)); + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + // Last ping and participant type are not seen as changed, even if they did. + expectedUnchanged.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId9", USER, "theUserId9")); + + verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } + + @Test + public void testAllParticipantsUpdateDisconnected() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onAllParticipantsUpdate(DISCONNECTED); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + InOrder inOrder = inOrder(mockedCallParticipantListObserver); + + inOrder.verify(mockedCallParticipantListObserver).onCallEndedForAll(); + inOrder.verify(mockedCallParticipantListObserver).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testAllParticipantsUpdateDisconnectedWithSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newUser(DISCONNECTED, 2, "theSessionId2", USER, "theUserId2")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 4, "theSessionId4", GUEST)); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onAllParticipantsUpdate(DISCONNECTED); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedLeft.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 4, "theSessionId4", GUEST)); + + InOrder inOrder = inOrder(mockedCallParticipantListObserver); + + inOrder.verify(mockedCallParticipantListObserver).onCallEndedForAll(); + inOrder.verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } + + @Test + public void testAllParticipantsUpdateDisconnectedNoOneInCall() { + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onAllParticipantsUpdate(DISCONNECTED); + + InOrder inOrder = inOrder(mockedCallParticipantListObserver); + + inOrder.verify(mockedCallParticipantListObserver).onCallEndedForAll(); + verifyNoMoreInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testAllParticipantsUpdateDisconnectedThenJoinCallAgain() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participantListMessageListener.onAllParticipantsUpdate(DISCONNECTED); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } +} diff --git a/app/src/test/java/com/nextcloud/talk/call/CallParticipantListInternalSignalingTest.java b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListInternalSignalingTest.java new file mode 100644 index 0000000000..fb393b10f2 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListInternalSignalingTest.java @@ -0,0 +1,535 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.models.json.participants.Participant; +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatcher; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.DISCONNECTED; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.IN_CALL; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.WITH_AUDIO; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.WITH_VIDEO; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.only; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +public class CallParticipantListInternalSignalingTest { + + private static class UsersInRoomParticipantBuilder { + private Participant newUser(long inCall, long lastPing, String sessionId, String userId) { + Participant participant = new Participant(); + participant.setInCall(inCall); + participant.setLastPing(lastPing); + participant.setSessionId(sessionId); + participant.setUserId(userId); + + return participant; + } + + private Participant newGuest(long inCall, long lastPing, String sessionId) { + Participant participant = new Participant(); + participant.setInCall(inCall); + participant.setLastPing(lastPing); + participant.setSessionId(sessionId); + + return participant; + } + } + + private final UsersInRoomParticipantBuilder builder = new UsersInRoomParticipantBuilder(); + + private CallParticipantList callParticipantList; + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener; + + private CallParticipantList.Observer mockedCallParticipantListObserver; + + private Collection expectedJoined; + private Collection expectedUpdated; + private Collection expectedLeft; + private Collection expectedUnchanged; + + // The order of the left participants in some tests depends on how they are internally sorted by the map, so the + // list of left participants needs to be checked ignoring the sorting (or, rather, sorting by session ID as in + // expectedLeft). + // Other tests can just relay on the not guaranteed, but known internal sorting of the elements. + private final ArgumentMatcher> matchesExpectedLeftIgnoringOrder = left -> { + Collections.sort(left, Comparator.comparing(Participant::getSessionId)); + return expectedLeft.equals(left); + }; + + @Before + public void setUp() { + SignalingMessageReceiver mockedSignalingMessageReceiver = mock(SignalingMessageReceiver.class); + + callParticipantList = new CallParticipantList(mockedSignalingMessageReceiver); + + mockedCallParticipantListObserver = mock(CallParticipantList.Observer.class); + + // Get internal ParticipantListMessageListener from callParticipantList set in the + // mockedSignalingMessageReceiver. + ArgumentCaptor participantListMessageListenerArgumentCaptor = + ArgumentCaptor.forClass(SignalingMessageReceiver.ParticipantListMessageListener.class); + + verify(mockedSignalingMessageReceiver).addListener(participantListMessageListenerArgumentCaptor.capture()); + + participantListMessageListener = participantListMessageListenerArgumentCaptor.getValue(); + + expectedJoined = new ArrayList<>(); + expectedUpdated = new ArrayList<>(); + expectedLeft = new ArrayList<>(); + expectedUnchanged = new ArrayList<>(); + } + + @Test + public void testUsersInRoomJoinRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomJoinRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(DISCONNECTED, 4, "theSessionId4", "theUserId4")); + participants.add(builder.newUser(DISCONNECTED, 5, "theSessionId5", "theUserId5")); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomJoinRoomThenJoinCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomJoinRoomThenJoinCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(DISCONNECTED, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + expectedJoined.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomJoinRoomAndCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomJoinRoomAndCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + expectedJoined.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomJoinRoomAndCallRepeated() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + participantListMessageListener.onUsersInRoom(participants); + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomChangeCallFlags() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedUpdated.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomChangeCallFlagsSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL | WITH_VIDEO, 4, "theSessionId4", "theUserId4")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedUpdated.add(builder.newUser(IN_CALL, 1, "theSessionId1", "theUserId1")); + expectedUpdated.add(builder.newUser(IN_CALL | WITH_VIDEO, 4, "theSessionId4", "theUserId4")); + expectedUnchanged.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomChangeLastPing() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomChangeLastPingSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 2, "theSessionId2")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", "theUserId3")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 108, "theSessionId2")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 815, "theSessionId3", "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomLeaveCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomLeaveCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomLeaveCallThenLeaveRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomLeaveCallThenLeaveRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomLeaveCallAndRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + + participantListMessageListener.onUsersInRoom(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomLeaveCallAndRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } + + @Test + public void testUsersInRoomSeveralEventsSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + participants.add(builder.newUser(IN_CALL, 5, "theSessionId5", "theUserId5")); + // theSessionId6 has not joined yet. + participants.add(builder.newGuest(IN_CALL | WITH_VIDEO, 7, "theSessionId7")); + participants.add(builder.newUser(DISCONNECTED, 8, "theSessionId8", "theUserId8")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 9, "theSessionId9", "theUserId9")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + // theSessionId1 is gone. + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 5, "theSessionId5", "theUserId5")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 6, "theSessionId6")); + participants.add(builder.newGuest(IN_CALL, 7, "theSessionId7")); + participants.add(builder.newUser(IN_CALL, 8, "theSessionId8", "theUserId8")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId9", "theUserId9")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newGuest(IN_CALL | WITH_AUDIO, 6, "theSessionId6")); + expectedJoined.add(builder.newUser(IN_CALL, 8, "theSessionId8", "theUserId8")); + expectedUpdated.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 5, "theSessionId5", "theUserId5")); + expectedUpdated.add(builder.newGuest(IN_CALL, 7, "theSessionId7")); + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + // Last ping is not seen as changed, even if it did. + expectedUnchanged.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId9", "theUserId9")); + + verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } +} diff --git a/app/src/test/java/com/nextcloud/talk/call/CallParticipantListTest.java b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListTest.java new file mode 100644 index 0000000000..7fed0f6d85 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListTest.java @@ -0,0 +1,61 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +public class CallParticipantListTest { + + private SignalingMessageReceiver mockedSignalingMessageReceiver; + + private CallParticipantList callParticipantList; + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener; + + @Before + public void setUp() { + mockedSignalingMessageReceiver = mock(SignalingMessageReceiver.class); + + callParticipantList = new CallParticipantList(mockedSignalingMessageReceiver); + + // Get internal ParticipantListMessageListener from callParticipantList set in the + // mockedSignalingMessageReceiver. + ArgumentCaptor participantListMessageListenerArgumentCaptor = + ArgumentCaptor.forClass(SignalingMessageReceiver.ParticipantListMessageListener.class); + + verify(mockedSignalingMessageReceiver).addListener(participantListMessageListenerArgumentCaptor.capture()); + + participantListMessageListener = participantListMessageListenerArgumentCaptor.getValue(); + } + + @Test + public void testDestroy() { + callParticipantList.destroy(); + + verify(mockedSignalingMessageReceiver).removeListener(participantListMessageListener); + verifyNoMoreInteractions(mockedSignalingMessageReceiver); + } +} From ed5e8fc82eabdfc78a1997e0c78bb6e920929659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 14:54:41 +0100 Subject: [PATCH 16/19] Use helper class to keep track of the participants in a call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As CallParticipantList starts listening on the signaling messages as soon as it is created it needs to be created and destroyed right before entering and exiting a call. Otherwise it could receive messages on other states (for example, while the "connection timeout" message is shown) and thus once the local participant joined the event would not include the other participants already in the call as joined (although they would be anyway reported as unchanged). Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index df89f8b0b2..967717fe6c 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -61,6 +61,7 @@ import com.nextcloud.talk.api.NcApi; import com.nextcloud.talk.application.NextcloudTalkApplication; import com.nextcloud.talk.call.CallParticipant; +import com.nextcloud.talk.call.CallParticipantList; import com.nextcloud.talk.call.CallParticipantModel; import com.nextcloud.talk.data.user.model.User; import com.nextcloud.talk.databinding.CallActivityBinding; @@ -125,8 +126,8 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -272,27 +273,22 @@ public class CallActivity extends CallBaseActivity { private Handler screenParticipantDisplayItemManagersHandler = new Handler(Looper.getMainLooper()); - private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { - - @Override - public void onUsersInRoom(List participants) { - processUsersInRoom(participants); - } - + private CallParticipantList.Observer callParticipantListObserver = new CallParticipantList.Observer() { @Override - public void onParticipantsUpdate(List participants) { - processUsersInRoom(participants); + public void onCallParticipantsChanged(Collection joined, Collection updated, + Collection left, Collection unchanged) { + handleCallParticipantsChanged(joined, updated, left, unchanged); } @Override - public void onAllParticipantsUpdate(long inCall) { - if (inCall == Participant.InCallFlags.DISCONNECTED) { - Log.d(TAG, "A moderator ended the call for all."); - hangup(true); - } + public void onCallEndedForAll() { + Log.d(TAG, "A moderator ended the call for all."); + hangup(true); } }; + private CallParticipantList callParticipantList; + private SignalingMessageReceiver.OfferMessageListener offerMessageListener = new SignalingMessageReceiver.OfferMessageListener() { @Override public void onOffer(String sessionId, String roomType, String sdp, String nick) { @@ -1245,7 +1241,6 @@ public void onAnimationEnd(Animator animation) { @Override public void onDestroy() { - signalingMessageReceiver.removeListener(participantListMessageListener); signalingMessageReceiver.removeListener(offerMessageListener); if (localStream != null) { @@ -1379,7 +1374,6 @@ public void onNext(@io.reactivex.annotations.NonNull CapabilitiesOverall capabil setupAndInitiateWebSocketsConnection(); } else { signalingMessageReceiver = internalSignalingMessageReceiver; - signalingMessageReceiver.addListener(participantListMessageListener); signalingMessageReceiver.addListener(offerMessageListener); signalingMessageSender = internalSignalingMessageSender; joinRoomAndCall(); @@ -1469,6 +1463,9 @@ private void performCall() { inCallFlag += Participant.InCallFlags.WITH_VIDEO; } + callParticipantList = new CallParticipantList(signalingMessageReceiver); + callParticipantList.addObserver(callParticipantListObserver); + int apiVersion = ApiUtils.getCallApiVersion(conversationUser, new int[]{ApiUtils.APIv4, 1}); ncApi.joinCall( @@ -1583,7 +1580,6 @@ private void setupAndInitiateWebSocketsConnection() { // Although setupAndInitiateWebSocketsConnection could be called several times the web socket is // initialized just once, so the message receiver is also initialized just once. signalingMessageReceiver = webSocketClient.getSignalingMessageReceiver(); - signalingMessageReceiver.addListener(participantListMessageListener); signalingMessageReceiver.addListener(offerMessageListener); signalingMessageSender = webSocketClient.getSignalingMessageSender(); } else { @@ -1741,6 +1737,9 @@ private void hangupNetworkCalls(boolean shutDownView) { Log.d(TAG, "hangupNetworkCalls. shutDownView=" + shutDownView); int apiVersion = ApiUtils.getCallApiVersion(conversationUser, new int[]{ApiUtils.APIv4, 1}); + callParticipantList.removeObserver(callParticipantListObserver); + callParticipantList.destroy(); + ncApi.leaveCall(credentials, ApiUtils.getUrlForCall(apiVersion, baseUrl, roomToken)) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) @@ -1778,11 +1777,9 @@ private void startVideoCapture() { } } - private void processUsersInRoom(List participants) { - Log.d(TAG, "processUsersInRoom"); - List newSessions = new ArrayList<>(); - Set oldSessions = new HashSet<>(); - Map userIdsBySessionId = new HashMap<>(); + private void handleCallParticipantsChanged(Collection joined, Collection updated, + Collection left, Collection unchanged) { + Log.d(TAG, "handleCallParticipantsChanged"); hasMCU = hasExternalSignalingServer && webSocketClient != null && webSocketClient.hasMCU(); Log.d(TAG, " hasMCU is " + hasMCU); @@ -1795,58 +1792,45 @@ private void processUsersInRoom(List participants) { Log.d(TAG, " currentSessionId is " + currentSessionId); + List participantsInCall = new ArrayList<>(); + participantsInCall.addAll(joined); + participantsInCall.addAll(updated); + participantsInCall.addAll(unchanged); + boolean isSelfInCall = false; - for (Participant participant : participants) { + for (Participant participant : participantsInCall) { long inCallFlag = participant.getInCall(); if (!participant.getSessionId().equals(currentSessionId)) { Log.d(TAG, " inCallFlag of participant " + participant.getSessionId().substring(0, 4) + " : " + inCallFlag); - - boolean isInCall = inCallFlag != 0; - if (isInCall) { - newSessions.add(participant.getSessionId()); - } - - userIdsBySessionId.put(participant.getSessionId(), participant.getUserId()); } else { Log.d(TAG, " inCallFlag of currentSessionId: " + inCallFlag); isSelfInCall = inCallFlag != 0; - if (inCallFlag == 0 && currentCallStatus != CallStatus.LEAVING && ApplicationWideCurrentRoomHolder.getInstance().isInCall()) { - Log.d(TAG, "Most probably a moderator ended the call for all."); - hangup(true); - - return; - } } } - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { - if (!peerConnectionWrapper.isMCUPublisher()) { - oldSessions.add(peerConnectionWrapper.getSessionId()); - } + if (!isSelfInCall && currentCallStatus != CallStatus.LEAVING && ApplicationWideCurrentRoomHolder.getInstance().isInCall()) { + Log.d(TAG, "Most probably a moderator ended the call for all."); + hangup(true); + + return; } if (!isSelfInCall) { Log.d(TAG, "Self not in call, disconnecting from all other sessions"); - for (String sessionId : oldSessions) { - Log.d(TAG, " oldSession that will be removed is: " + sessionId); + for (Participant participant : participantsInCall) { + String sessionId = participant.getSessionId(); + Log.d(TAG, " session that will be removed is: " + sessionId); endPeerConnection(sessionId, false); } return; } - // Calculate sessions that left the call - List disconnectedSessions = new ArrayList<>(oldSessions); - disconnectedSessions.removeAll(newSessions); - - // Calculate sessions that join the call - newSessions.removeAll(oldSessions); - if (currentCallStatus == CallStatus.LEAVING) { return; } @@ -1856,11 +1840,25 @@ private void processUsersInRoom(List participants) { getOrCreatePeerConnectionWrapperForSessionIdAndType(webSocketClient.getSessionId(), VIDEO_STREAM_TYPE_VIDEO, true); } - for (String sessionId : newSessions) { + boolean selfJoined = false; + + for (Participant participant : joined) { + String sessionId = participant.getSessionId(); + + if (sessionId == null) { + Log.w(TAG, "Null sessionId for call participant, this should not happen: " + participant); + continue; + } + + if (sessionId.equals(currentSessionId)) { + selfJoined = true; + continue; + } + Log.d(TAG, " newSession joined: " + sessionId); getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, VIDEO_STREAM_TYPE_VIDEO, false); - String userId = userIdsBySessionId.get(sessionId); + String userId = participant.getUserId(); if (userId != null) { callParticipants.get(sessionId).setUserId(userId); } @@ -1874,11 +1872,13 @@ private void processUsersInRoom(List participants) { callParticipants.get(sessionId).setNick(nick); } - if (newSessions.size() > 0 && currentCallStatus != CallStatus.IN_CONVERSATION) { + boolean othersInCall = selfJoined ? joined.size() > 1 : joined.size() > 0; + if (othersInCall && currentCallStatus != CallStatus.IN_CONVERSATION) { setCallState(CallStatus.IN_CONVERSATION); } - for (String sessionId : disconnectedSessions) { + for (Participant participant : left) { + String sessionId = participant.getSessionId(); Log.d(TAG, " oldSession that will be removed is: " + sessionId); endPeerConnection(sessionId, false); } From 2cb7572dbc7191c3a3205b98fc8d06ccd9ee2294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 17:14:41 +0100 Subject: [PATCH 17/19] Extract methods to add and remove call participants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 92 +++++++++++-------- 1 file changed, 52 insertions(+), 40 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 967717fe6c..43705eccc3 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1973,31 +1973,7 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp if (!publisher) { CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant == null) { - callParticipant = new CallParticipant(sessionId); - callParticipants.put(sessionId, callParticipant); - - SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener = - new CallActivityCallParticipantMessageListener(sessionId); - callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); - signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); - - if (!hasExternalSignalingServer) { - OfferAnswerNickProvider offerAnswerNickProvider = new OfferAnswerNickProvider(sessionId); - offerAnswerNickProviders.put(sessionId, offerAnswerNickProvider); - signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); - signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); - } - - final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); - - ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = - new ScreenParticipantDisplayItemManager(callParticipantModel); - screenParticipantDisplayItemManagers.put(sessionId, screenParticipantDisplayItemManager); - callParticipantModel.addObserver(screenParticipantDisplayItemManager, screenParticipantDisplayItemManagersHandler); - - runOnUiThread(() -> { - addParticipantDisplayItem(callParticipantModel, "video"); - }); + callParticipant = addCallParticipant(sessionId); } if ("screen".equals(type)) { @@ -2017,6 +1993,36 @@ private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndTyp } } + private CallParticipant addCallParticipant(String sessionId) { + CallParticipant callParticipant = new CallParticipant(sessionId); + callParticipants.put(sessionId, callParticipant); + + SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener = + new CallActivityCallParticipantMessageListener(sessionId); + callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); + signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); + + if (!hasExternalSignalingServer) { + OfferAnswerNickProvider offerAnswerNickProvider = new OfferAnswerNickProvider(sessionId); + offerAnswerNickProviders.put(sessionId, offerAnswerNickProvider); + signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); + signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); + } + + final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); + + ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = + new ScreenParticipantDisplayItemManager(callParticipantModel); + screenParticipantDisplayItemManagers.put(sessionId, screenParticipantDisplayItemManager); + callParticipantModel.addObserver(screenParticipantDisplayItemManager, screenParticipantDisplayItemManagersHandler); + + runOnUiThread(() -> { + addParticipantDisplayItem(callParticipantModel, "video"); + }); + + return callParticipant; + } + private List getPeerConnectionWrapperListForSessionId(String sessionId) { List internalList = new ArrayList<>(); for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { @@ -2055,26 +2061,32 @@ private void endPeerConnection(String sessionId, boolean justScreen) { } if (!justScreen) { - CallParticipant callParticipant = callParticipants.remove(sessionId); - if (callParticipant != null) { - ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = - screenParticipantDisplayItemManagers.remove(sessionId); - callParticipant.getCallParticipantModel().removeObserver(screenParticipantDisplayItemManager); + removeCallParticipant(sessionId); + } + } - callParticipant.destroy(); + private void removeCallParticipant(String sessionId) { + CallParticipant callParticipant = callParticipants.remove(sessionId); + if (callParticipant == null) { + return; + } - SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); - signalingMessageReceiver.removeListener(listener); + ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = + screenParticipantDisplayItemManagers.remove(sessionId); + callParticipant.getCallParticipantModel().removeObserver(screenParticipantDisplayItemManager); - OfferAnswerNickProvider offerAnswerNickProvider = offerAnswerNickProviders.remove(sessionId); - if (offerAnswerNickProvider != null) { - signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); - signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); - } + callParticipant.destroy(); - runOnUiThread(() -> removeParticipantDisplayItem(sessionId, "video")); - } + SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); + signalingMessageReceiver.removeListener(listener); + + OfferAnswerNickProvider offerAnswerNickProvider = offerAnswerNickProviders.remove(sessionId); + if (offerAnswerNickProvider != null) { + signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); + signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); } + + runOnUiThread(() -> removeParticipantDisplayItem(sessionId, "video")); } private void removeParticipantDisplayItem(String sessionId, String videoStreamType) { From 9ae969b0f82c30909c0189fb5d3432d897b9908b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 20:34:32 +0100 Subject: [PATCH 18/19] Split call participants and peer connections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of trying to create a video peer connection for any joined participant now only a call participant is created for any joined participant, and a video peer connection is created only for those participants that are publishing audio or video. If a call participants does not have a video peer connection the call participant is now seen as "connected" from the UI, as there is no need to show a progress bar for that participant. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 46 +++++++++++++++---- .../talk/adapters/ParticipantDisplayItem.java | 5 +- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 43705eccc3..ee07afe0bb 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1721,14 +1721,22 @@ private void hangup(boolean shutDownView) { } } - List sessionIdsToEnd = new ArrayList(peerConnectionWrapperList.size()); + List peerConnectionIdsToEnd = new ArrayList(peerConnectionWrapperList.size()); for (PeerConnectionWrapper wrapper : peerConnectionWrapperList) { - sessionIdsToEnd.add(wrapper.getSessionId()); + peerConnectionIdsToEnd.add(wrapper.getSessionId()); } - for (String sessionId : sessionIdsToEnd) { + for (String sessionId : peerConnectionIdsToEnd) { endPeerConnection(sessionId, false); } + List callParticipantIdsToEnd = new ArrayList(peerConnectionWrapperList.size()); + for (CallParticipant callParticipant : callParticipants.values()) { + callParticipantIdsToEnd.add(callParticipant.getCallParticipantModel().getSessionId()); + } + for (String sessionId : callParticipantIdsToEnd) { + removeCallParticipant(sessionId); + } + hangupNetworkCalls(shutDownView); ApplicationWideCurrentRoomHolder.getInstance().setInCall(false); } @@ -1798,6 +1806,7 @@ private void handleCallParticipantsChanged(Collection joined, Colle participantsInCall.addAll(unchanged); boolean isSelfInCall = false; + Participant selfParticipant = null; for (Participant participant : participantsInCall) { long inCallFlag = participant.getInCall(); @@ -1809,6 +1818,7 @@ private void handleCallParticipantsChanged(Collection joined, Colle } else { Log.d(TAG, " inCallFlag of currentSessionId: " + inCallFlag); isSelfInCall = inCallFlag != 0; + selfParticipant = participant; } } @@ -1826,6 +1836,7 @@ private void handleCallParticipantsChanged(Collection joined, Colle String sessionId = participant.getSessionId(); Log.d(TAG, " session that will be removed is: " + sessionId); endPeerConnection(sessionId, false); + removeCallParticipant(sessionId); } return; @@ -1841,6 +1852,7 @@ private void handleCallParticipantsChanged(Collection joined, Colle } boolean selfJoined = false; + boolean selfParticipantHasAudioOrVideo = participantInCallFlagsHaveAudioOrVideo(selfParticipant); for (Participant participant : joined) { String sessionId = participant.getSessionId(); @@ -1856,7 +1868,8 @@ private void handleCallParticipantsChanged(Collection joined, Colle } Log.d(TAG, " newSession joined: " + sessionId); - getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, VIDEO_STREAM_TYPE_VIDEO, false); + + CallParticipant callParticipant = addCallParticipant(sessionId); String userId = participant.getUserId(); if (userId != null) { @@ -1870,6 +1883,17 @@ private void handleCallParticipantsChanged(Collection joined, Colle nick = offerAnswerNickProviders.get(sessionId) != null ? offerAnswerNickProviders.get(sessionId).getNick() : ""; } callParticipants.get(sessionId).setNick(nick); + + boolean participantHasAudioOrVideo = participantInCallFlagsHaveAudioOrVideo(participant); + + // FIXME Without MCU, PeerConnectionWrapper only sends an offer if the local session ID is higher than the + // remote session ID. However, if the other participant does not have audio nor video that participant + // will not send an offer, so no connection is actually established when the remote participant has a + // higher session ID but is not publishing media. + if ((hasMCU && participantHasAudioOrVideo) || + (!hasMCU && selfParticipantHasAudioOrVideo && (!participantHasAudioOrVideo || sessionId.compareTo(currentSessionId) < 0))) { + getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, VIDEO_STREAM_TYPE_VIDEO, false); + } } boolean othersInCall = selfJoined ? joined.size() > 1 : joined.size() > 0; @@ -1881,7 +1905,17 @@ private void handleCallParticipantsChanged(Collection joined, Colle String sessionId = participant.getSessionId(); Log.d(TAG, " oldSession that will be removed is: " + sessionId); endPeerConnection(sessionId, false); + removeCallParticipant(sessionId); + } + } + + private boolean participantInCallFlagsHaveAudioOrVideo(Participant participant) { + if (participant == null) { + return false; } + + return (participant.getInCall() & Participant.InCallFlags.WITH_AUDIO) > 0 || + (!isVoiceOnlyCall && (participant.getInCall() & Participant.InCallFlags.WITH_VIDEO) > 0); } private void deletePeerConnection(PeerConnectionWrapper peerConnectionWrapper) { @@ -2059,10 +2093,6 @@ private void endPeerConnection(String sessionId, boolean justScreen) { } } } - - if (!justScreen) { - removeCallParticipant(sessionId); - } } private void removeCallParticipant(String sessionId) { diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 6a9912eadb..418afa1a5f 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -95,7 +95,10 @@ private void updateUrlForAvatar() { public boolean isConnected() { return iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || - iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; + iceConnectionState == PeerConnection.IceConnectionState.COMPLETED || + // If there is no connection state that means that no connection is needed, so it is a special case that is + // also seen as "connected". + iceConnectionState == null; } public String getNick() { From 67e259f79257fe5cd7679f872f7546737d2d5fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 20:49:29 +0100 Subject: [PATCH 19/19] Simplify ending the peer connections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The peer connections will be of either "video" or "screen" type, so they can be simply removed based on the session id and an explicit type. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 63 +++++++------------ 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index ee07afe0bb..49fcb6d6a1 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1726,7 +1726,8 @@ private void hangup(boolean shutDownView) { peerConnectionIdsToEnd.add(wrapper.getSessionId()); } for (String sessionId : peerConnectionIdsToEnd) { - endPeerConnection(sessionId, false); + endPeerConnection(sessionId, "video"); + endPeerConnection(sessionId, "screen"); } List callParticipantIdsToEnd = new ArrayList(peerConnectionWrapperList.size()); @@ -1835,7 +1836,8 @@ private void handleCallParticipantsChanged(Collection joined, Colle for (Participant participant : participantsInCall) { String sessionId = participant.getSessionId(); Log.d(TAG, " session that will be removed is: " + sessionId); - endPeerConnection(sessionId, false); + endPeerConnection(sessionId, "video"); + endPeerConnection(sessionId, "screen"); removeCallParticipant(sessionId); } @@ -1904,7 +1906,8 @@ private void handleCallParticipantsChanged(Collection joined, Colle for (Participant participant : left) { String sessionId = participant.getSessionId(); Log.d(TAG, " oldSession that will be removed is: " + sessionId); - endPeerConnection(sessionId, false); + endPeerConnection(sessionId, "video"); + endPeerConnection(sessionId, "screen"); removeCallParticipant(sessionId); } } @@ -1918,11 +1921,6 @@ private boolean participantInCallFlagsHaveAudioOrVideo(Participant participant) (!isVoiceOnlyCall && (participant.getInCall() & Participant.InCallFlags.WITH_VIDEO) > 0); } - private void deletePeerConnection(PeerConnectionWrapper peerConnectionWrapper) { - peerConnectionWrapper.removePeerConnection(); - peerConnectionWrapperList.remove(peerConnectionWrapper); - } - private PeerConnectionWrapper getPeerConnectionWrapperForSessionIdAndType(String sessionId, String type) { for (PeerConnectionWrapper wrapper : peerConnectionWrapperList) { if (wrapper.getSessionId().equals(sessionId) @@ -2057,42 +2055,27 @@ private CallParticipant addCallParticipant(String sessionId) { return callParticipant; } - private List getPeerConnectionWrapperListForSessionId(String sessionId) { - List internalList = new ArrayList<>(); - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { - if (peerConnectionWrapper.getSessionId().equals(sessionId)) { - internalList.add(peerConnectionWrapper); - } + private void endPeerConnection(String sessionId, String type) { + PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(sessionId, type); + if (peerConnectionWrapper == null) { + return; } - return internalList; - } - - private void endPeerConnection(String sessionId, boolean justScreen) { - List peerConnectionWrappers; - if (!(peerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty()) { - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrappers) { - if (peerConnectionWrapper.getSessionId().equals(sessionId)) { - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - peerConnectionWrapper.removeObserver(selfPeerConnectionObserver); - } - - String videoStreamType = peerConnectionWrapper.getVideoStreamType(); - if (VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType) || !justScreen) { - CallParticipant callParticipant = callParticipants.get(sessionId); - if (callParticipant != null) { - if ("screen".equals(videoStreamType)) { - callParticipant.setScreenPeerConnectionWrapper(null); - } else { - callParticipant.setPeerConnectionWrapper(null); - } - } + if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { + peerConnectionWrapper.removeObserver(selfPeerConnectionObserver); + } - deletePeerConnection(peerConnectionWrapper); - } - } + CallParticipant callParticipant = callParticipants.get(sessionId); + if (callParticipant != null) { + if ("screen".equals(type)) { + callParticipant.setScreenPeerConnectionWrapper(null); + } else { + callParticipant.setPeerConnectionWrapper(null); } } + + peerConnectionWrapper.removePeerConnection(); + peerConnectionWrapperList.remove(peerConnectionWrapper); } private void removeCallParticipant(String sessionId) { @@ -2599,7 +2582,7 @@ public CallActivityCallParticipantMessageListener(String sessionId) { @Override public void onUnshareScreen() { - endPeerConnection(sessionId, true); + endPeerConnection(sessionId, "screen"); } }