Skip to content

Commit 87ca9ed

Browse files
code-ashercode-asher
committed
Refactor retry
1 parent aa27ca1 commit 87ca9ed

File tree

3 files changed

+97
-86
lines changed

3 files changed

+97
-86
lines changed

‎src/main/kotlin/com/coder/gateway/CoderGatewayConnectionProvider.kt

Lines changed: 38 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package com.coder.gateway
44

55
import com.coder.gateway.sdk.humanizeDuration
6+
import com.coder.gateway.sdk.isCancellation
67
import com.coder.gateway.sdk.isWorkerTimeout
78
import com.coder.gateway.sdk.suspendingRetryWithExponentialBackOff
89
import com.coder.gateway.services.CoderRecentWorkspaceConnectionsService
@@ -23,6 +24,7 @@ import kotlinx.coroutines.launch
2324
import net.schmizz.sshj.common.SSHException
2425
import net.schmizz.sshj.connection.ConnectionException
2526
import java.time.Duration
27+
import java.util.concurrent.TimeUnit
2628
import java.util.concurrent.TimeoutException
2729

2830
class CoderGatewayConnectionProvider : GatewayConnectionProvider {
@@ -33,45 +35,49 @@ class CoderGatewayConnectionProvider : GatewayConnectionProvider {
3335
// TODO: If this fails determine if it is an auth error and if so prompt
3436
// for a new token, configure the CLI, then try again.
3537
clientLifetime.launchUnderBackgroundProgress(CoderGatewayBundle.message("gateway.connector.coder.connection.provider.title"), canBeCancelled = true, isIndeterminate = true, project = null) {
36-
val context = suspendingRetryWithExponentialBackOff(
37-
label = "connect",
38-
logger = logger,
39-
action = { attempt ->
40-
logger.info("Deploying (attempt $attempt)...")
41-
indicator.text =
42-
if (attempt > 1) CoderGatewayBundle.message("gateway.connector.coder.connection.retry.text", attempt)
43-
else CoderGatewayBundle.message("gateway.connector.coder.connection.loading.text")
44-
SshMultistagePanelContext(parameters.toHostDeployInputs())
45-
},
46-
predicate = { e ->
47-
e is ConnectionException || e is TimeoutException
48-
|| e is SSHException || e is DeployException
49-
},
50-
update = { _, e, remainingMs ->
51-
val message =
52-
if (isWorkerTimeout(e)) "Failed to upload worker binary...it may have timed out"
53-
else e.message ?: CoderGatewayBundle.message("gateway.connector.no-details")
54-
if (remainingMs != null) {
55-
indicator.text = message
56-
indicator.text = CoderGatewayBundle.message("gateway.connector.coder.connection.retry-error.text", humanizeDuration(remainingMs))
57-
} else {
58-
ApplicationManager.getApplication().invokeAndWait {
59-
Messages.showMessageDialog(
60-
message,
61-
CoderGatewayBundle.message("gateway.connector.coder.connection.error.text"),
62-
Messages.getErrorIcon())
63-
}
64-
}
65-
},
66-
)
67-
if (context != null) {
38+
try {
39+
val context = suspendingRetryWithExponentialBackOff(
40+
action = { attempt ->
41+
logger.info("Connecting... (attempt $attempt")
42+
SshMultistagePanelContext(parameters.toHostDeployInputs())
43+
},
44+
retryIf = {
45+
it is ConnectionException || it is TimeoutException
46+
|| it is SSHException || it is DeployException
47+
},
48+
onError = { attempt, nextMs, e ->
49+
logger.error("Failed to connect (attempt $attempt; will retry in $nextMs ms)")
50+
indicator.text =
51+
if (isWorkerTimeout(e)) "Failed to upload worker binary...it may have timed out"
52+
else e.message ?: CoderGatewayBundle.message("gateway.connector.no-details")
53+
},
54+
onCountdown = { attempt, remainingMs ->
55+
indicator.text =
56+
if (remainingMs == null) CoderGatewayBundle.message("gateway.connector.coder.connection.loading.text")
57+
else if (remainingMs < TimeUnit.SECONDS.toMillis(1)) CoderGatewayBundle.message("gateway.connector.coder.connection.retry.text", attempt)
58+
else CoderGatewayBundle.message("gateway.connector.coder.connection.retry-error.text", humanizeDuration(remainingMs))
59+
},
60+
)
6861
launch {
6962
logger.info("Deploying and starting IDE with $context")
7063
// At this point JetBrains takes over with their own UI.
7164
@Suppress("UnstableApiUsage") SshDeployFlowUtil.fullDeployCycle(
7265
clientLifetime, context, Duration.ofMinutes(10)
7366
)
7467
}
68+
} catch (e: Exception) {
69+
if (isCancellation(e)) {
70+
logger.info("Connection canceled due to ${e.javaClass}")
71+
} else {
72+
// The dialog will close once we return so write the error
73+
// out into a new dialog.
74+
ApplicationManager.getApplication().invokeAndWait {
75+
Messages.showMessageDialog(
76+
e.message ?: CoderGatewayBundle.message("gateway.connector.no-details"),
77+
CoderGatewayBundle.message("gateway.connector.coder.connection.error.text"),
78+
Messages.getErrorIcon())
79+
}
80+
}
7581
}
7682
}
7783

‎src/main/kotlin/com/coder/gateway/sdk/Retry.kt

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package com.coder.gateway.sdk
22

3-
import com.intellij.openapi.diagnostic.Logger
43
import com.intellij.openapi.progress.ProcessCanceledException
54
import com.intellij.ssh.SshException
65
import com.jetbrains.gateway.ssh.deploy.DeployException
@@ -28,19 +27,19 @@ fun unwrap(ex: Exception): Throwable {
2827
* things like duplicate logs can be avoided). If remaining is null then no
2928
* more retries will be attempted.
3029
*
31-
* If an exception related to canceling is received then return null.
30+
* If an unretryable exception is received (including those related to
31+
* canceling) that exception (after being unwrapped) will be thrown.
3232
*/
3333
suspend fun <T> suspendingRetryWithExponentialBackOff(
3434
initialDelayMs: Long = TimeUnit.SECONDS.toMillis(5),
3535
backOffLimitMs: Long = TimeUnit.MINUTES.toMillis(3),
3636
backOffFactor: Int = 2,
3737
backOffJitter: Double = 0.1,
38-
label: String,
39-
logger: Logger,
40-
predicate: (e: Throwable) -> Boolean,
41-
update: (attempt: Int, e: Throwable, remaining: Long?) -> Unit,
42-
action: suspend (attempt: Int) -> T?
43-
): T? {
38+
retryIf: (e: Throwable) -> Boolean,
39+
onError: (attempt: Int, nextMs: Long, e: Throwable) -> Unit,
40+
onCountdown: (attempt: Int, remaining: Long?) -> Unit,
41+
action: suspend (attempt: Int) -> T
42+
): T {
4443
val random = Random()
4544
var delayMs = initialDelayMs
4645
for (attempt in 1..Int.MAX_VALUE) {
@@ -51,23 +50,13 @@ suspend fun <T> suspendingRetryWithExponentialBackOff(
5150
// SshException can happen due to anything from a timeout to being
5251
// canceled so unwrap to find out.
5352
val unwrappedEx = if (originalEx is SshException) unwrap(originalEx) else originalEx
54-
when (unwrappedEx) {
55-
is InterruptedException,
56-
is CancellationException,
57-
is ProcessCanceledException -> {
58-
logger.info("Retrying $label canceled due to ${unwrappedEx.javaClass}")
59-
return null
60-
}
53+
if (!retryIf(unwrappedEx)) {
54+
throw unwrappedEx
6155
}
62-
if (!predicate(unwrappedEx)) {
63-
logger.error("Failed to $label (attempt $attempt; will not retry)", originalEx)
64-
update(attempt, unwrappedEx, null)
65-
return null
66-
}
67-
logger.error("Failed to $label (attempt $attempt; will retry in $delayMs ms)", originalEx)
56+
onError(attempt, delayMs, unwrappedEx)
6857
var remainingMs = delayMs
6958
while (remainingMs > 0) {
70-
update(attempt, unwrappedEx, remainingMs)
59+
onCountdown(attempt, remainingMs)
7160
val next = min(remainingMs, TimeUnit.SECONDS.toMillis(1))
7261
remainingMs -= next
7362
delay(next)
@@ -98,3 +87,12 @@ fun humanizeDuration(durationMs: Long): String {
9887
fun isWorkerTimeout(e: Throwable): Boolean {
9988
return e is DeployException && e.message.contains("Worker binary deploy failed")
10089
}
90+
91+
/**
92+
* Return true if the exception is some kind of cancellation.
93+
*/
94+
fun isCancellation(e: Throwable): Boolean {
95+
return e is InterruptedException
96+
|| e is CancellationException
97+
|| e is ProcessCanceledException
98+
}

‎src/main/kotlin/com/coder/gateway/views/steps/CoderLocateRemoteProjectStepView.kt

Lines changed: 39 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import com.coder.gateway.sdk.CoderCLIManager
99
import com.coder.gateway.sdk.CoderRestClientService
1010
import com.coder.gateway.sdk.OS
1111
import com.coder.gateway.sdk.humanizeDuration
12+
import com.coder.gateway.sdk.isCancellation
1213
import com.coder.gateway.sdk.isWorkerTimeout
1314
import com.coder.gateway.sdk.suspendingRetryWithExponentialBackOff
1415
import com.coder.gateway.sdk.toURL
@@ -70,6 +71,7 @@ import net.schmizz.sshj.connection.ConnectionException
7071
import java.awt.Component
7172
import java.awt.FlowLayout
7273
import java.util.Locale
74+
import java.util.concurrent.TimeUnit
7375
import java.util.concurrent.TimeoutException
7476
import javax.swing.ComboBoxModel
7577
import javax.swing.DefaultComboBoxModel
@@ -162,6 +164,7 @@ class CoderLocateRemoteProjectStepView(private val setNextButtonEnabled: (Boolea
162164
// Clear contents from the last attempt if any.
163165
cbIDEComment.foreground = UIUtil.getContextHelpForeground()
164166
cbIDEComment.text = CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.ide.none.comment")
167+
cbIDE.renderer = IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.loading.text"))
165168
ideComboBoxModel.removeAllElements()
166169
setNextButtonEnabled(false)
167170

@@ -178,42 +181,46 @@ class CoderLocateRemoteProjectStepView(private val setNextButtonEnabled: (Boolea
178181
terminalLink.url = coderClient.coderURL.withPath("/@${coderClient.me.username}/${selectedWorkspace.name}/terminal").toString()
179182

180183
ideResolvingJob = cs.launch {
181-
val ides = suspendingRetryWithExponentialBackOff(
182-
label = "retrieve IDEs",
183-
logger = logger,
184-
action={ attempt ->
185-
logger.info("Deploying to ${selectedWorkspace.name} on $deploymentURL (attempt $attempt)")
186-
// Reset text in the select dropdown.
187-
withContext(Dispatchers.Main) {
188-
cbIDE.renderer = IDECellRenderer(
189-
if (attempt > 1) CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.retry.text", attempt)
190-
else CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.loading.text"))
191-
}
192-
val executor = createRemoteExecutor(CoderCLIManager.getHostName(deploymentURL, selectedWorkspace))
193-
if (ComponentValidator.getInstance(tfProject).isEmpty) {
194-
installRemotePathValidator(executor)
195-
}
196-
retrieveIDEs(executor, selectedWorkspace)
197-
},
198-
predicate = { e ->
199-
e is ConnectionException || e is TimeoutException
200-
|| e is SSHException || e is DeployException
201-
},
202-
update = { _, e, remainingMs ->
203-
cbIDEComment.foreground = UIUtil.getErrorForeground()
204-
cbIDEComment.text =
205-
if (isWorkerTimeout(e)) "Failed to upload worker binary...it may have timed out. Check the command log for more details."
206-
else e.message ?: CoderGatewayBundle.message("gateway.connector.no-details")
207-
cbIDE.renderer =
208-
if (remainingMs != null) IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.retry-error.text", humanizeDuration(remainingMs)))
209-
else IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.error.text"), UIUtil.getBalloonErrorIcon())
210-
},
211-
)
212-
if (ides != null) {
184+
try {
185+
val ides = suspendingRetryWithExponentialBackOff(
186+
action ={ attempt ->
187+
logger.info("Retrieving IDEs...(attempt $attempt)")
188+
val executor = createRemoteExecutor(CoderCLIManager.getHostName(deploymentURL, selectedWorkspace))
189+
if (ComponentValidator.getInstance(tfProject).isEmpty) {
190+
installRemotePathValidator(executor)
191+
}
192+
retrieveIDEs(executor, selectedWorkspace)
193+
},
194+
retryIf = {
195+
it is ConnectionException || it is TimeoutException
196+
|| it is SSHException || it is DeployException
197+
},
198+
onError = { attempt, nextMs, e ->
199+
logger.error("Failed to retrieve IDEs (attempt $attempt; will retry in $nextMs ms)")
200+
cbIDEComment.foreground = UIUtil.getErrorForeground()
201+
cbIDEComment.text =
202+
if (isWorkerTimeout(e)) "Failed to upload worker binary...it may have timed out. Check the command log for more details."
203+
else e.message ?: CoderGatewayBundle.message("gateway.connector.no-details")
204+
},
205+
onCountdown = { attempt, remainingMs ->
206+
cbIDE.renderer =
207+
if (remainingMs == null) IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.error.text"), UIUtil.getBalloonErrorIcon())
208+
else if (remainingMs < TimeUnit.SECONDS.toMillis(1)) IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.retry.text", attempt))
209+
else IDECellRenderer(CoderGatewayBundle.message("gateway.connector.view.coder.remoteproject.retry-error.text", humanizeDuration(remainingMs)))
210+
},
211+
)
213212
withContext(Dispatchers.Main) {
214213
ideComboBoxModel.addAll(ides)
215214
cbIDE.selectedIndex = 0
216215
}
216+
} catch (e: Exception) {
217+
if (isCancellation(e)) {
218+
logger.info("Connection canceled due to ${e.javaClass}")
219+
} else {
220+
logger.error("Failed to retrieve IDEs (will not retry)")
221+
cbIDEComment.foreground = UIUtil.getErrorForeground()
222+
cbIDEComment.text = e.message ?: CoderGatewayBundle.message("gateway.connector.no-details")
223+
}
217224
}
218225
}
219226
}

0 commit comments

Comments
 (0)